[llvm] cbe6493 - [AMDGPU] Enable true16 pattern to build vectors (0, vgpr) (#191896)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 15 13:49:35 PDT 2026
Author: Stanislav Mekhanoshin
Date: 2026-04-15T13:49:28-07:00
New Revision: cbe64931e791ba565b4dea3eab30135326531777
URL: https://github.com/llvm/llvm-project/commit/cbe64931e791ba565b4dea3eab30135326531777
DIFF: https://github.com/llvm/llvm-project/commit/cbe64931e791ba565b4dea3eab30135326531777.diff
LOG: [AMDGPU] Enable true16 pattern to build vectors (0, vgpr) (#191896)
Fixes: https://github.com/llvm/llvm-project/issues/190796
Added:
Modified:
llvm/lib/Target/AMDGPU/SIInstructions.td
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
llvm/test/CodeGen/AMDGPU/atomicrmw-bf16-gfx11plus.ll
llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
llvm/test/CodeGen/AMDGPU/bf16-math.ll
llvm/test/CodeGen/AMDGPU/bf16.ll
llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll
llvm/test/CodeGen/AMDGPU/fabs.bf16.ll
llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.bf16.ll
llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
llvm/test/CodeGen/AMDGPU/fdiv.bf16.ll
llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll
llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll
llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll
llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
llvm/test/CodeGen/AMDGPU/global-saddr-load.ll
llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll
llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll
llvm/test/CodeGen/AMDGPU/llvm.log.ll
llvm/test/CodeGen/AMDGPU/llvm.log10.ll
llvm/test/CodeGen/AMDGPU/llvm.log2.ll
llvm/test/CodeGen/AMDGPU/mad-mix-bf16.ll
llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll
llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
llvm/test/CodeGen/AMDGPU/minmax3-tree-reduction.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 008fdbaddfd22..2dbed449dcfb2 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -3816,6 +3816,13 @@ def : GCNPat <
>;
}
+let True16Predicate = UseRealTrue16Insts in {
+def : GCNPat <
+ (v2i16 (DivergentBinFrag<build_vector> (i16 0), (i16 VGPR_16:$src1))),
+ (v2i16 (V_LSHLREV_B32_e64 (i16 16), VGPR_16:$src1))
+>;
+}
+
def : GCNPat <
(v2i16 (UniformBinFrag<build_vector> (i16 SReg_32:$src1), (i16 0))),
(S_AND_B32 (S_MOV_B32 (i32 0xffff)), SReg_32:$src1)
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
index 94f4be1c32418..2b4e9f26c958a 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
@@ -2711,74 +2711,76 @@ define <4 x i32> @bitcast_v8bf16_to_v4i32(<8 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v2, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v13, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v5, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v10, v13 :: v_dual_and_b32 v3, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v10, v1, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v6, v7 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.h
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v11, v12, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v14, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v5, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v5, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v9
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v2, v10 :: v_dual_add_f32 v4, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v1, v9 :: v_dual_and_b32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v3, v9 :: v_dual_and_b32 v0, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v11, v0, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v2, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.h
; GFX11-TRUE16-NEXT: .LBB22_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -7211,74 +7213,76 @@ define <4 x float> @bitcast_v8bf16_to_v4f32(<8 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v2, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v13, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v5, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v10, v13 :: v_dual_and_b32 v3, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v10, v1, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v6, v7 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.h
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v11, v12, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v14, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v5, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v5, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v9
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v2, v10 :: v_dual_add_f32 v4, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v1, v9 :: v_dual_and_b32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v3, v9 :: v_dual_and_b32 v0, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v11, v0, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v2, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.h
; GFX11-TRUE16-NEXT: .LBB46_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -11346,74 +11350,76 @@ define <2 x i64> @bitcast_v8bf16_to_v2i64(<8 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB66_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v2, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v13, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v5, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v10, v13 :: v_dual_and_b32 v3, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v10, v1, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v6, v7 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.h
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v11, v12, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v14, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v5, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v5, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v9
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v2, v10 :: v_dual_add_f32 v4, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v1, v9 :: v_dual_and_b32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v3, v9 :: v_dual_and_b32 v0, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v11, v0, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v2, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.h
; GFX11-TRUE16-NEXT: .LBB66_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -15082,74 +15088,76 @@ define <2 x double> @bitcast_v8bf16_to_v2f64(<8 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v2, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v13, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v5, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v10, v13 :: v_dual_and_b32 v3, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v10, v1, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v6, v7 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.h
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v11, v12, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v14, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v5, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v5, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v9
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v2, v10 :: v_dual_add_f32 v4, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v1, v9 :: v_dual_and_b32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v3, v9 :: v_dual_and_b32 v0, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v11, v0, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v2, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.h
; GFX11-TRUE16-NEXT: .LBB82_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -18474,75 +18482,76 @@ define <8 x i16> @bitcast_v8bf16_to_v8i16(<8 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB94_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v6, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v5
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v5, v7 :: v_dual_add_f32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v11, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v5.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v7, v8 :: v_dual_and_b32 v1, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v12, v5, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v9, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.l
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_lshlrev_b32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v7, 16, 1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v7, v8 :: v_dual_and_b32 v2, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v4 :: v_dual_add_f32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v10, v9, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v2, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v6, v8, v9 :: v_dual_and_b32 v3, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v2, 0x40c00000, v2
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v10, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v11, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v11, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v10, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v11, v7, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v9, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.h
; GFX11-TRUE16-NEXT: .LBB94_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -21417,73 +21426,75 @@ define <8 x half> @bitcast_v8bf16_to_v8f16(<8 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB102_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v1.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v2.l
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v7, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v7, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v1, v7 :: v_dual_and_b32 v7, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v11, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v9, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v3.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v7, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v8, v8, v12 :: v_dual_and_b32 v3, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_add_f32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v11, v12, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v13, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v8, v9 :: v_dual_and_b32 v6, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_lshlrev_b32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v7 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v10, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v8, v12 :: v_dual_and_b32 v3, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_cndmask_b32 v10, v10, v14
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v13, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v10, v14, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v13, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v8.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v8, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v10.h
; GFX11-TRUE16-NEXT: .LBB102_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
@@ -24190,78 +24201,78 @@ define <16 x i8> @bitcast_v8bf16_to_v16i8(<8 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v6 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v8, v10 :: v_dual_and_b32 v1, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v13, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v0, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v11, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v4.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v3, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v7
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v13, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v13
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v8, v13, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v14, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v4.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v11, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v3, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v7, v9, 0x7fff
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v3, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v0, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v11, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v11, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v6.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[22:23], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 8, v16
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v21, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v7, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v21, v3, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v12.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v1, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 8, v16
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v0, v1, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v8.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v21
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
index b6b321a08f7aa..dd9fa1456e37a 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
@@ -663,8 +663,7 @@ define i16 @bitcast_bf16_to_i16(bfloat %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB6_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
@@ -1137,8 +1136,7 @@ define half @bitcast_bf16_to_f16(bfloat %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB10_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
index 547985e7ef4e3..5697cb1e909c8 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
@@ -149,35 +149,35 @@ define <3 x half> @bitcast_v3bf16_to_v3f16(<3 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB0_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v3, v7 :: v_dual_and_b32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0x7fc0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
; GFX11-TRUE16-NEXT: .LBB0_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -866,38 +866,36 @@ define <3 x i16> @bitcast_v3bf16_to_v3i16(<3 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB4_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, 0x7fc0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
; GFX11-TRUE16-NEXT: .LBB4_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
index 45e835ddb0e28..aef9e4bd95135 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
@@ -2174,40 +2174,42 @@ define i64 @bitcast_v4bf16_to_i64(<4 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v2, 0x40c00000, v2
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v7, v10 :: v_dual_add_f32 v0, 0x40c00000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v1, v5, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.h
; GFX11-TRUE16-NEXT: .LBB22_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -5343,40 +5345,42 @@ define double @bitcast_v4bf16_to_f64(<4 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v2, 0x40c00000, v2
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v7, v10 :: v_dual_add_f32 v0, 0x40c00000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v1, v5, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.h
; GFX11-TRUE16-NEXT: .LBB46_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -8207,40 +8211,42 @@ define <2 x i32> @bitcast_v4bf16_to_v2i32(<4 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB66_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v2, 0x40c00000, v2
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v7, v10 :: v_dual_add_f32 v0, 0x40c00000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v1, v5, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.h
; GFX11-TRUE16-NEXT: .LBB66_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -10755,40 +10761,42 @@ define <2 x float> @bitcast_v4bf16_to_v2f32(<4 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v2, 0x40c00000, v2
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v7, v10 :: v_dual_add_f32 v0, 0x40c00000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v1, v5, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.h
; GFX11-TRUE16-NEXT: .LBB82_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -12965,42 +12973,42 @@ define <4 x i16> @bitcast_v4bf16_to_v4i16(<4 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB94_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v3, v5 :: v_dual_add_f32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v9, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v9, v11 :: v_dual_add_f32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h
; GFX11-TRUE16-NEXT: .LBB94_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -14801,42 +14809,44 @@ define <4 x half> @bitcast_v4bf16_to_v4f16(<4 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB102_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v7, v10 :: v_dual_and_b32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v9.h
; GFX11-TRUE16-NEXT: .LBB102_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -16488,38 +16498,43 @@ define <8 x i8> @bitcast_v4bf16_to_v8i8(<4 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v2 :: v_dual_add_f32 v2, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v12, v8, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v1, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v10, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v12, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v6, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v8
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
index 687bea385a266..6ee3282633f03 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
@@ -2014,57 +2014,57 @@ define <3 x i32> @bitcast_v6bf16_to_v3i32(<6 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB10_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v5 :: v_dual_lshlrev_b32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v11, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v0, v1, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v0, v8 :: v_dual_add_f32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v7.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v11
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v10, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v9, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v8, v11 :: v_dual_and_b32 v2, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v5, v7 :: v_dual_lshlrev_b32 v0, 16, v0
; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v3, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v1, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v8, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_cndmask_b32 v1, v2, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v0, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v7.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v10.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v7, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.h
; GFX11-TRUE16-NEXT: .LBB10_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -5188,57 +5188,57 @@ define <3 x float> @bitcast_v6bf16_to_v3f32(<6 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB26_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v5 :: v_dual_lshlrev_b32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v11, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v0, v1, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v0, v8 :: v_dual_add_f32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v7.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v11
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v10, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v9, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v8, v11 :: v_dual_and_b32 v2, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v5, v7 :: v_dual_lshlrev_b32 v0, 16, v0
; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v3, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v1, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v8, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_cndmask_b32 v1, v2, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v0, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v7.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v10.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v7, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.h
; GFX11-TRUE16-NEXT: .LBB26_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -7769,61 +7769,64 @@ define <12 x i8> @bitcast_v6bf16_to_v12i8(<6 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB38_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v17, 0x7fc07fc0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v4, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v12, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_add_f32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v12, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v11, v0, v8 :: v_dual_add_f32 v0, 0x40c00000, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v12
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v10, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v3, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v10, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v4, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v11, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v3, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v12, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v4.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v1, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_cndmask_b32 v10, v9, v10
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v3.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v8.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[12:13], 24, v[10:11]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[12:13], 24, v[10:11]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v8.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[14:15], 24, v[16:17]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
; GFX11-TRUE16-NEXT: .LBB38_4: ; %end
@@ -11441,55 +11444,56 @@ define <6 x half> @bitcast_v6bf16_to_v6f16(<6 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB48_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v6, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_cndmask_b32 v3, v1, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v2, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v10, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v9, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v12, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v6, v8 :: v_dual_add_f32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v11, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v8, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
; GFX11-TRUE16-NEXT: v_add3_u32 v9, v11, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v7.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
; GFX11-TRUE16-NEXT: .LBB48_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
@@ -12501,59 +12505,61 @@ define <6 x i16> @bitcast_v6bf16_to_v6i16(<6 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB52_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v6, v7 :: v_dual_and_b32 v1, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v4, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v9, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v1 :: v_dual_add_f32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v8, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v8, v9, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v9, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v8, v9 :: v_dual_and_b32 v2, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v7, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v5.h
; GFX11-TRUE16-NEXT: .LBB52_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/atomicrmw-bf16-gfx11plus.ll b/llvm/test/CodeGen/AMDGPU/atomicrmw-bf16-gfx11plus.ll
index 535f05bc01b42..e784c33918f9b 100644
--- a/llvm/test/CodeGen/AMDGPU/atomicrmw-bf16-gfx11plus.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomicrmw-bf16-gfx11plus.ll
@@ -11,22 +11,25 @@ define amdgpu_kernel void @v_atomicrmw_fadd_bf16(ptr addrspace(1) %out, i1 %in,
; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
; GFX11-TRUE16-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v2, v0, s[0:1] offset:4
+; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[0:1] offset:4
; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, -4
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, 3
; GFX11-TRUE16-NEXT: s_load_b32 s3, s[0:1], 0x0
; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 3
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_lshl_b32 s4, 0xffff, s2
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s3
; GFX11-TRUE16-NEXT: s_not_b32 s3, s4
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -34,7 +37,7 @@ define amdgpu_kernel void @v_atomicrmw_fadd_bf16(ptr addrspace(1) %out, i1 %in,
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v3
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -42,8 +45,8 @@ define amdgpu_kernel void @v_atomicrmw_fadd_bf16(ptr addrspace(1) %out, i1 %in,
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s3, v0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v0, v4, v[0:1], s[0:1] glc
diff --git a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
index e1da8a7a52a51..4b917977acd58 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16-conversions.ll
@@ -12,20 +12,11 @@ define amdgpu_ps float @v_test_cvt_bf16_f32_v(bfloat %v) {
; GCN-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GCN-NEXT: ; return to shader part epilog
;
-; GFX1250-FAKE16-LABEL: v_test_cvt_bf16_f32_v:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250-FAKE16-NEXT: ; return to shader part epilog
-;
-; GFX1250-REAL16-LABEL: v_test_cvt_bf16_f32_v:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v0, v1
-; GFX1250-REAL16-NEXT: ; return to shader part epilog
+; GFX1250-LABEL: v_test_cvt_bf16_f32_v:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: ; return to shader part epilog
%cvt = fpext bfloat %v to float
ret float %cvt
}
@@ -141,22 +132,13 @@ define amdgpu_ps float @v_test_cvt_f32_bf16_v(float %src) {
; GFX-950-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX-950-NEXT: ; return to shader part epilog
;
-; GFX1250-FAKE16-LABEL: v_test_cvt_f32_bf16_v:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250-FAKE16-NEXT: ; return to shader part epilog
-;
-; GFX1250-REAL16-LABEL: v_test_cvt_f32_bf16_v:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-REAL16-NEXT: v_cvt_pk_bf16_f32 v1, v0, s0
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.l, 0
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX1250-REAL16-NEXT: ; return to shader part epilog
+; GFX1250-LABEL: v_test_cvt_f32_bf16_v:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: ; return to shader part epilog
%trunc = fptrunc float %src to bfloat
%ext = fpext bfloat %trunc to float
ret float %ext
@@ -651,3 +633,6 @@ entry:
declare float @llvm.fabs.f32(float)
declare double @llvm.fabs.f64(double)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX1250-FAKE16: {{.*}}
+; GFX1250-REAL16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/bf16-math.ll b/llvm/test/CodeGen/AMDGPU/bf16-math.ll
index 73f02c6d18704..4434d8a496a27 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16-math.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16-math.ll
@@ -408,10 +408,9 @@ define amdgpu_ps float @test_clamp_bf16_hi16(<2 x bfloat> %src) {
; TRUE16: ; %bb.0:
; TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; TRUE16-NEXT: v_pk_max_num_bf16 v1, v0, v0 op_sel_hi:[0,0] clamp
-; TRUE16-NEXT: v_mov_b16_e32 v0.l, 0
-; TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; TRUE16-NEXT: v_pk_max_num_bf16 v0, v0, v0 op_sel_hi:[0,0] clamp
+; TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; TRUE16-NEXT: ; return to shader part epilog
;
; FAKE16-LABEL: test_clamp_bf16_hi16:
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index 32bb3debed76a..8863244c1e91b 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -9549,19 +9549,17 @@ define bfloat @v_fadd_bf16(bfloat %a, bfloat %b) {
; GFX11TRUE16-LABEL: v_fadd_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v1, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -13699,10 +13697,9 @@ define bfloat @v_fadd_bf16_fpimm_0(bfloat %arg0) {
; GFX11TRUE16-LABEL: v_fadd_bf16_fpimm_0:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add_f32_e32 v0, 1.0, v1
+; GFX11TRUE16-NEXT: v_add_f32_e32 v0, 1.0, v0
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -13811,10 +13808,9 @@ define bfloat @v_fadd_bf16_fpimm_1(bfloat %arg0) {
; GFX11TRUE16-LABEL: v_fadd_bf16_fpimm_1:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add_f32_e32 v0, 0x42280000, v1
+; GFX11TRUE16-NEXT: v_add_f32_e32 v0, 0x42280000, v0
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -13929,19 +13925,17 @@ define bfloat @v_fsub_bf16(bfloat %a, bfloat %b) {
; GFX11TRUE16-LABEL: v_fsub_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11TRUE16-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -14754,19 +14748,17 @@ define bfloat @v_fmul_bf16(bfloat %a, bfloat %b) {
; GFX11TRUE16-LABEL: v_fmul_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, v1, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -18968,34 +18960,32 @@ define bfloat @v_fdiv_bf16(bfloat %a, bfloat %b) {
; GFX11TRUE16-LABEL: v_fdiv_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11TRUE16-NEXT: v_div_scale_f32 v1, null, v0, v0, v2
-; GFX11TRUE16-NEXT: v_div_scale_f32 v5, vcc_lo, v2, v0, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_rcp_f32_e32 v3, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_div_scale_f32 v2, null, v1, v1, v0
+; GFX11TRUE16-NEXT: v_rcp_f32_e32 v3, v2
; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11TRUE16-NEXT: v_fma_f32 v4, -v1, v3, 1.0
+; GFX11TRUE16-NEXT: v_fma_f32 v4, -v2, v3, 1.0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v3, v4, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_div_scale_f32 v5, vcc_lo, v0, v1, v0
; GFX11TRUE16-NEXT: v_mul_f32_e32 v4, v5, v3
-; GFX11TRUE16-NEXT: v_fma_f32 v6, -v1, v4, v5
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_fma_f32 v6, -v2, v4, v5
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v4, v6, v3
-; GFX11TRUE16-NEXT: v_fma_f32 v1, -v1, v4, v5
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_div_fmas_f32 v1, v1, v3, v4
-; GFX11TRUE16-NEXT: v_div_fixup_f32 v0, v1, v0, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_fma_f32 v2, -v2, v4, v5
+; GFX11TRUE16-NEXT: v_div_fmas_f32 v2, v2, v3, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_div_fixup_f32 v0, v2, v1, v0
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -19031,60 +19021,31 @@ define bfloat @v_fdiv_bf16(bfloat %a, bfloat %b) {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250TRUE16-LABEL: v_fdiv_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX1250TRUE16-NEXT: v_div_scale_f32 v1, null, v0, v0, v2
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
-; GFX1250TRUE16-NEXT: v_rcp_f32_e32 v3, v1
-; GFX1250TRUE16-NEXT: v_nop
-; GFX1250TRUE16-NEXT: v_fma_f32 v4, -v1, v3, 1.0
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_fmac_f32_e32 v3, v4, v3
-; GFX1250TRUE16-NEXT: v_div_scale_f32 v4, vcc_lo, v2, v0, v2
-; GFX1250TRUE16-NEXT: v_mul_f32_e32 v5, v4, v3
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_fma_f32 v6, -v1, v5, v4
-; GFX1250TRUE16-NEXT: v_fmac_f32_e32 v5, v6, v3
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_fma_f32 v1, -v1, v5, v4
-; GFX1250TRUE16-NEXT: v_div_fmas_f32 v1, v1, v3, v5
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_div_fixup_f32 v0, v1, v0, v2
-; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250FAKE16-LABEL: v_fdiv_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_div_scale_f32 v2, null, v1, v1, v0
-; GFX1250FAKE16-NEXT: v_rcp_f32_e32 v3, v2
-; GFX1250FAKE16-NEXT: v_nop
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_fma_f32 v4, -v2, v3, 1.0
-; GFX1250FAKE16-NEXT: v_fmac_f32_e32 v3, v4, v3
-; GFX1250FAKE16-NEXT: v_div_scale_f32 v4, vcc_lo, v0, v1, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_mul_f32_e32 v5, v4, v3
-; GFX1250FAKE16-NEXT: v_fma_f32 v6, -v2, v5, v4
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_fmac_f32_e32 v5, v6, v3
-; GFX1250FAKE16-NEXT: v_fma_f32 v2, -v2, v5, v4
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_div_fmas_f32 v2, v2, v3, v5
-; GFX1250FAKE16-NEXT: v_div_fixup_f32 v0, v2, v1, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_fdiv_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_div_scale_f32 v2, null, v1, v1, v0
+; GFX1250-NEXT: v_rcp_f32_e32 v3, v2
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_fma_f32 v4, -v2, v3, 1.0
+; GFX1250-NEXT: v_fmac_f32_e32 v3, v4, v3
+; GFX1250-NEXT: v_div_scale_f32 v4, vcc_lo, v0, v1, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_mul_f32_e32 v5, v4, v3
+; GFX1250-NEXT: v_fma_f32 v6, -v2, v5, v4
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_fmac_f32_e32 v5, v6, v3
+; GFX1250-NEXT: v_fma_f32 v2, -v2, v5, v4
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_div_fmas_f32 v2, v2, v3, v5
+; GFX1250-NEXT: v_div_fixup_f32 v0, v2, v1, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = fdiv bfloat %a, %b
ret bfloat %op
}
@@ -19189,7 +19150,7 @@ define amdgpu_ps i32 @s_fabs_bf16(bfloat inreg %a) {
;
; GFX1250-LABEL: s_fabs_bf16:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_and_b32 s0, s0, 0x7fff
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_and_b32 s0, 0xffff, s0
@@ -19304,7 +19265,7 @@ define amdgpu_ps i32 @s_fneg_bf16(bfloat inreg %a) {
;
; GFX1250-LABEL: s_fneg_bf16:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_xor_b32 s0, s0, 0x8000
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_and_b32 s0, 0xffff, s0
@@ -19418,7 +19379,7 @@ define amdgpu_ps i32 @s_fneg_fabs_bf16(bfloat inreg %a) {
;
; GFX1250-LABEL: s_fneg_fabs_bf16:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_bitset1_b32 s0, 15
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_and_b32 s0, 0xffff, s0
@@ -19518,19 +19479,17 @@ define bfloat @v_minnum_bf16(bfloat %a, bfloat %b) {
; GFX11TRUE16-LABEL: v_minnum_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11TRUE16-NEXT: v_min_f32_e32 v0, v1, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_min_f32_e32 v0, v0, v1
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -19551,29 +19510,15 @@ define bfloat @v_minnum_bf16(bfloat %a, bfloat %b) {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250TRUE16-LABEL: v_minnum_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX1250TRUE16-NEXT: v_min_num_f32_e32 v0, v1, v2
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250FAKE16-LABEL: v_minnum_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_min_num_f32_e32 v0, v0, v1
-; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_minnum_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_min_num_f32_e32 v0, v0, v1
+; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = call bfloat @llvm.minnum.bf16(bfloat %a, bfloat %b)
ret bfloat %op
}
@@ -23962,19 +23907,17 @@ define bfloat @v_maxnum_bf16(bfloat %a, bfloat %b) {
; GFX11TRUE16-LABEL: v_maxnum_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11TRUE16-NEXT: v_max_f32_e32 v0, v1, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_max_f32_e32 v0, v0, v1
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -23995,29 +23938,15 @@ define bfloat @v_maxnum_bf16(bfloat %a, bfloat %b) {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250TRUE16-LABEL: v_maxnum_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX1250TRUE16-NEXT: v_max_num_f32_e32 v0, v1, v2
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250FAKE16-LABEL: v_maxnum_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_max_num_f32_e32 v0, v0, v1
-; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_maxnum_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_max_num_f32_e32 v0, v0, v1
+; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = call bfloat @llvm.maxnum.bf16(bfloat %a, bfloat %b)
ret bfloat %op
}
@@ -28494,12 +28423,11 @@ define bfloat @v_sqrt_bf16(bfloat %a) {
; GFX11TRUE16-LABEL: v_sqrt_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xf800000, v1
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, 0x4f800000, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, 0x4f800000, v0
+; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xf800000, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_sqrt_f32_e32 v1, v0
; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
@@ -28848,12 +28776,11 @@ define bfloat @v_rsq_bf16(bfloat %x) {
; GFX11TRUE16-LABEL: v_rsq_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xf800000, v1
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, 0x4f800000, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, 0x4f800000, v0
+; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xf800000, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_sqrt_f32_e32 v1, v0
; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
@@ -29253,12 +29180,11 @@ define bfloat @v_neg_rsq_bf16(bfloat %x) {
; GFX11TRUE16-LABEL: v_neg_rsq_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xf800000, v1
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, 0x4f800000, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, 0x4f800000, v0
+; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xf800000, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_sqrt_f32_e32 v1, v0
; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
@@ -29470,10 +29396,9 @@ define bfloat @v_ldexp_bf16_i32(bfloat %a, i32 %b) {
; GFX11TRUE16-LABEL: v_ldexp_bf16_i32:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v2, v1
+; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -29500,26 +29425,15 @@ define bfloat @v_ldexp_bf16_i32(bfloat %a, i32 %b) {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250TRUE16-LABEL: v_ldexp_bf16_i32:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_ldexp_f32 v0, v2, v1
-; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250FAKE16-LABEL: v_ldexp_bf16_i32:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_ldexp_bf16_i32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = call bfloat @llvm.ldexp.bf16.i32(bfloat %a, i32 %b)
ret bfloat %op
}
@@ -29606,18 +29520,17 @@ define { bfloat, i16 } @v_frexp_bf16_i16(bfloat %a) {
; GFX11TRUE16-LABEL: v_frexp_bf16_i16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_frexp_mant_f32_e32 v0, v1
-; GFX11TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
; GFX11TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -29638,28 +29551,16 @@ define { bfloat, i16 } @v_frexp_bf16_i16(bfloat %a) {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250TRUE16-LABEL: v_frexp_bf16_i16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1250TRUE16-NEXT: v_frexp_mant_f32_e32 v0, v1
-; GFX1250TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
-; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250FAKE16-LABEL: v_frexp_bf16_i16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1250FAKE16-NEXT: v_frexp_mant_f32_e32 v0, v1
-; GFX1250FAKE16-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
-; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_frexp_bf16_i16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_frexp_mant_f32_e32 v0, v1
+; GFX1250-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = call { bfloat, i16 } @llvm.frexp.bf16.i16(bfloat %a)
ret { bfloat, i16 } %op
}
@@ -29837,12 +29738,11 @@ define bfloat @v_log_bf16(bfloat %a) {
; GFX11TRUE16-LABEL: v_log_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 32, vcc_lo
-; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v1, v0
+; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo
+; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_log_f32_e32 v0, v0
; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
@@ -29897,58 +29797,31 @@ define bfloat @v_log_bf16(bfloat %a) {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250TRUE16-LABEL: v_log_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v1
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 32, vcc_lo
-; GFX1250TRUE16-NEXT: v_ldexp_f32 v0, v1, v0
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
-; GFX1250TRUE16-NEXT: v_log_f32_e32 v0, v0
-; GFX1250TRUE16-NEXT: v_nop
-; GFX1250TRUE16-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0
-; GFX1250TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_fma_f32 v2, 0x3f317217, v0, -v1
-; GFX1250TRUE16-NEXT: v_fmamk_f32 v2, v0, 0x3377d1cf, v2
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250FAKE16-LABEL: v_log_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo
-; GFX1250FAKE16-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
-; GFX1250FAKE16-NEXT: v_log_f32_e32 v0, v0
-; GFX1250FAKE16-NEXT: v_nop
-; GFX1250FAKE16-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0
-; GFX1250FAKE16-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_fma_f32 v2, 0x3f317217, v0, -v1
-; GFX1250FAKE16-NEXT: v_fmamk_f32 v2, v0, 0x3377d1cf, v2
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0
-; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_log_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
+; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo
+; GFX1250-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; GFX1250-NEXT: v_log_f32_e32 v0, v0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0
+; GFX1250-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_fma_f32 v2, 0x3f317217, v0, -v1
+; GFX1250-NEXT: v_fmamk_f32 v2, v0, 0x3377d1cf, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0
+; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = call bfloat @llvm.log.bf16(bfloat %a)
ret bfloat %op
}
@@ -30063,14 +29936,13 @@ define bfloat @v_log2_bf16(bfloat %a) {
; GFX11TRUE16-LABEL: v_log2_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 32, vcc_lo
-; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v1, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 32, vcc_lo
; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_log_f32_e32 v0, v0
; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1
@@ -30292,12 +30164,11 @@ define bfloat @v_log10_bf16(bfloat %a) {
; GFX11TRUE16-LABEL: v_log10_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 32, vcc_lo
-; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v1, v0
+; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo
+; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_log_f32_e32 v0, v0
; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
@@ -30352,58 +30223,31 @@ define bfloat @v_log10_bf16(bfloat %a) {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250TRUE16-LABEL: v_log10_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v1
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 32, vcc_lo
-; GFX1250TRUE16-NEXT: v_ldexp_f32 v0, v1, v0
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
-; GFX1250TRUE16-NEXT: v_log_f32_e32 v0, v0
-; GFX1250TRUE16-NEXT: v_nop
-; GFX1250TRUE16-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0
-; GFX1250TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_fma_f32 v2, 0x3e9a209a, v0, -v1
-; GFX1250TRUE16-NEXT: v_fmamk_f32 v2, v0, 0x3284fbcf, v2
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250FAKE16-LABEL: v_log10_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo
-; GFX1250FAKE16-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
-; GFX1250FAKE16-NEXT: v_log_f32_e32 v0, v0
-; GFX1250FAKE16-NEXT: v_nop
-; GFX1250FAKE16-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0
-; GFX1250FAKE16-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_fma_f32 v2, 0x3e9a209a, v0, -v1
-; GFX1250FAKE16-NEXT: v_fmamk_f32 v2, v0, 0x3284fbcf, v2
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0
-; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_log10_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
+; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo
+; GFX1250-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; GFX1250-NEXT: v_log_f32_e32 v0, v0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0
+; GFX1250-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_fma_f32 v2, 0x3e9a209a, v0, -v1
+; GFX1250-NEXT: v_fmamk_f32 v2, v0, 0x3284fbcf, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0
+; GFX1250-NEXT: v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = call bfloat @llvm.log10.bf16(bfloat %a)
ret bfloat %op
}
@@ -30582,26 +30426,25 @@ define bfloat @v_exp_bf16(bfloat %a) {
; GFX11TRUE16-LABEL: v_exp_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v1
-; GFX11TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc2ce8ed0, v1
-; GFX11TRUE16-NEXT: v_fma_f32 v2, 0x3fb8aa3b, v1, -v0
-; GFX11TRUE16-NEXT: v_rndne_f32_e32 v3, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0
+; GFX11TRUE16-NEXT: v_rndne_f32_e32 v2, v1
+; GFX11TRUE16-NEXT: v_fma_f32 v3, 0x3fb8aa3b, v0, -v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_fmamk_f32 v2, v1, 0x32a5705f, v2
-; GFX11TRUE16-NEXT: v_sub_f32_e32 v0, v0, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX11TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v3
-; GFX11TRUE16-NEXT: v_exp_f32_e32 v0, v0
+; GFX11TRUE16-NEXT: v_sub_f32_e32 v1, v1, v2
+; GFX11TRUE16-NEXT: v_fmamk_f32 v3, v0, 0x32a5705f, v3
+; GFX11TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX11TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc2ce8ed0, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX11TRUE16-NEXT: v_exp_f32_e32 v1, v1
; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v2
+; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v2
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x42b17218, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v0, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x42b17218, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
@@ -30650,30 +30493,29 @@ define bfloat @v_exp_bf16(bfloat %a) {
; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX1250TRUE16-NEXT: s_mov_b32 s0, 0x3fb8aa3b
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1250TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v1, s0, neg(0) op_sel_hi:[1,0,0]
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc2ce8ed0, v0
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v3, v1, s0, -v2 op_sel_hi:[1,0,0]
; GFX1250TRUE16-NEXT: v_rndne_f32_e32 v4, v2
; GFX1250TRUE16-NEXT: s_mov_b32 s0, 0x32a5705f
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v1, v1, s0, v3 op_sel_hi:[1,0,0]
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX1250TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc2ce8ed0, v3
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1250TRUE16-NEXT: v_add_f32_e32 v1, v2, v1
; GFX1250TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v4
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
; GFX1250TRUE16-NEXT: v_exp_f32_e32 v1, v1
; GFX1250TRUE16-NEXT: v_nop
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_ldexp_f32 v0, v1, v2
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
-; GFX1250TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x42b17218, v3
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v0, vcc_lo
+; GFX1250TRUE16-NEXT: v_ldexp_f32 v1, v1, v2
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250TRUE16-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
+; GFX1250TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x42b17218, v0
+; GFX1250TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
@@ -30824,14 +30666,13 @@ define bfloat @v_exp2_bf16(bfloat %a) {
; GFX11TRUE16-LABEL: v_exp2_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 0x42800000, vcc_lo
-; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v1, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_exp_f32_e32 v0, v0
; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
@@ -31055,26 +30896,25 @@ define bfloat @v_exp10_bf16(bfloat %a) {
; GFX11TRUE16-LABEL: v_exp10_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, 0x40549a78, v1
-; GFX11TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc23369f4, v1
-; GFX11TRUE16-NEXT: v_fma_f32 v2, 0x40549a78, v1, -v0
-; GFX11TRUE16-NEXT: v_rndne_f32_e32 v3, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0
+; GFX11TRUE16-NEXT: v_rndne_f32_e32 v2, v1
+; GFX11TRUE16-NEXT: v_fma_f32 v3, 0x40549a78, v0, -v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_fmamk_f32 v2, v1, 0x33979a37, v2
-; GFX11TRUE16-NEXT: v_sub_f32_e32 v0, v0, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX11TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v3
-; GFX11TRUE16-NEXT: v_exp_f32_e32 v0, v0
+; GFX11TRUE16-NEXT: v_sub_f32_e32 v1, v1, v2
+; GFX11TRUE16-NEXT: v_fmamk_f32 v3, v0, 0x33979a37, v3
+; GFX11TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX11TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc23369f4, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX11TRUE16-NEXT: v_exp_f32_e32 v1, v1
; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v2
+; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v2
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x421a209b, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v0, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x421a209b, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
@@ -31123,30 +30963,29 @@ define bfloat @v_exp10_bf16(bfloat %a) {
; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX1250TRUE16-NEXT: s_mov_b32 s0, 0x40549a78
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1250TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v1, s0, neg(0) op_sel_hi:[1,0,0]
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc23369f4, v0
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v3, v1, s0, -v2 op_sel_hi:[1,0,0]
; GFX1250TRUE16-NEXT: v_rndne_f32_e32 v4, v2
; GFX1250TRUE16-NEXT: s_mov_b32 s0, 0x33979a37
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v1, v1, s0, v3 op_sel_hi:[1,0,0]
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX1250TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc23369f4, v3
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1250TRUE16-NEXT: v_add_f32_e32 v1, v2, v1
; GFX1250TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v4
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
; GFX1250TRUE16-NEXT: v_exp_f32_e32 v1, v1
; GFX1250TRUE16-NEXT: v_nop
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_ldexp_f32 v0, v1, v2
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
-; GFX1250TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x421a209b, v3
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v0, vcc_lo
+; GFX1250TRUE16-NEXT: v_ldexp_f32 v1, v1, v2
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250TRUE16-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
+; GFX1250TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x421a209b, v0
+; GFX1250TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
@@ -31254,10 +31093,9 @@ define bfloat @v_ceil_bf16(bfloat %a) {
; GFX11TRUE16-LABEL: v_ceil_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_ceil_f32_e32 v0, v1
+; GFX11TRUE16-NEXT: v_ceil_f32_e32 v0, v0
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -31284,26 +31122,15 @@ define bfloat @v_ceil_bf16(bfloat %a) {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250TRUE16-LABEL: v_ceil_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_ceil_f32_e32 v0, v1
-; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250FAKE16-LABEL: v_ceil_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_ceil_f32_e32 v0, v0
-; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_ceil_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_ceil_f32_e32 v0, v0
+; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = call bfloat @llvm.ceil.bf16(bfloat %a)
ret bfloat %op
}
@@ -31379,10 +31206,9 @@ define bfloat @v_trunc_bf16(bfloat %a) {
; GFX11TRUE16-LABEL: v_trunc_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_trunc_f32_e32 v0, v1
+; GFX11TRUE16-NEXT: v_trunc_f32_e32 v0, v0
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -31409,26 +31235,15 @@ define bfloat @v_trunc_bf16(bfloat %a) {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250TRUE16-LABEL: v_trunc_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_trunc_f32_e32 v0, v1
-; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250FAKE16-LABEL: v_trunc_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_trunc_f32_e32 v0, v0
-; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_trunc_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_trunc_f32_e32 v0, v0
+; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = call bfloat @llvm.trunc.bf16(bfloat %a)
ret bfloat %op
}
@@ -31504,10 +31319,9 @@ define bfloat @v_rint_bf16(bfloat %a) {
; GFX11TRUE16-LABEL: v_rint_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_rndne_f32_e32 v0, v1
+; GFX11TRUE16-NEXT: v_rndne_f32_e32 v0, v0
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -31534,26 +31348,15 @@ define bfloat @v_rint_bf16(bfloat %a) {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250TRUE16-LABEL: v_rint_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_rndne_f32_e32 v0, v1
-; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250FAKE16-LABEL: v_rint_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_rndne_f32_e32 v0, v0
-; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_rint_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_rndne_f32_e32 v0, v0
+; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = call bfloat @llvm.rint.bf16(bfloat %a)
ret bfloat %op
}
@@ -31629,10 +31432,9 @@ define bfloat @v_nearbyint_bf16(bfloat %a) {
; GFX11TRUE16-LABEL: v_nearbyint_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_rndne_f32_e32 v0, v1
+; GFX11TRUE16-NEXT: v_rndne_f32_e32 v0, v0
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -31659,26 +31461,15 @@ define bfloat @v_nearbyint_bf16(bfloat %a) {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250TRUE16-LABEL: v_nearbyint_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_rndne_f32_e32 v0, v1
-; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250FAKE16-LABEL: v_nearbyint_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_rndne_f32_e32 v0, v0
-; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_nearbyint_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_rndne_f32_e32 v0, v0
+; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = call bfloat @llvm.nearbyint.bf16(bfloat %a)
ret bfloat %op
}
@@ -31790,17 +31581,16 @@ define bfloat @v_round_bf16(bfloat %a) {
; GFX11TRUE16-LABEL: v_round_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_trunc_f32_e32 v0, v1
-; GFX11TRUE16-NEXT: v_sub_f32_e32 v2, v1, v0
+; GFX11TRUE16-NEXT: v_trunc_f32_e32 v1, v0
+; GFX11TRUE16-NEXT: v_sub_f32_e32 v2, v0, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_cmp_ge_f32_e64 s0, |v2|, 0.5
; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, s0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0x7fffffff, v2, v1
-; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fffffff, v2, v0
+; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v1, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
@@ -31834,42 +31624,23 @@ define bfloat @v_round_bf16(bfloat %a) {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250TRUE16-LABEL: v_round_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_trunc_f32_e32 v2, v1
-; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v0, v2, -1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,1]
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_cmp_ge_f32_e64 s0, |v0|, 0.5
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s0
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_bfi_b32 v0, 0x7fffffff, v0, v1
-; GFX1250TRUE16-NEXT: v_add_f32_e32 v0, v2, v0
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250FAKE16-LABEL: v_round_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_trunc_f32_e32 v2, v1
-; GFX1250FAKE16-NEXT: v_fma_mix_f32_bf16 v0, v2, -1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,1]
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_cmp_ge_f32_e64 s0, |v0|, 0.5
-; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_bfi_b32 v0, 0x7fffffff, v0, v1
-; GFX1250FAKE16-NEXT: v_add_f32_e32 v0, v2, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_round_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_trunc_f32_e32 v2, v1
+; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, v2, -1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,1]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_cmp_ge_f32_e64 s0, |v0|, 0.5
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_bfi_b32 v0, 0x7fffffff, v0, v1
+; GFX1250-NEXT: v_add_f32_e32 v0, v2, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = call bfloat @llvm.round.bf16(bfloat %a)
ret bfloat %op
}
@@ -31945,10 +31716,9 @@ define bfloat @v_roundeven_bf16(bfloat %a) {
; GFX11TRUE16-LABEL: v_roundeven_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_rndne_f32_e32 v0, v1
+; GFX11TRUE16-NEXT: v_rndne_f32_e32 v0, v0
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -31975,26 +31745,15 @@ define bfloat @v_roundeven_bf16(bfloat %a) {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250TRUE16-LABEL: v_roundeven_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_rndne_f32_e32 v0, v1
-; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250FAKE16-LABEL: v_roundeven_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_rndne_f32_e32 v0, v0
-; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_roundeven_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_rndne_f32_e32 v0, v0
+; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = call bfloat @llvm.roundeven.bf16(bfloat %a)
ret bfloat %op
}
@@ -32070,10 +31829,9 @@ define bfloat @v_floor_bf16(bfloat %a) {
; GFX11TRUE16-LABEL: v_floor_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_floor_f32_e32 v0, v1
+; GFX11TRUE16-NEXT: v_floor_f32_e32 v0, v0
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -32100,26 +31858,15 @@ define bfloat @v_floor_bf16(bfloat %a) {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250TRUE16-LABEL: v_floor_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_floor_f32_e32 v0, v1
-; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250FAKE16-LABEL: v_floor_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_floor_f32_e32 v0, v0
-; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_floor_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_floor_f32_e32 v0, v0
+; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = call bfloat @llvm.floor.bf16(bfloat %a)
ret bfloat %op
}
@@ -32195,10 +31942,9 @@ define bfloat @v_canonicalize_bf16(bfloat %a) {
; GFX11TRUE16-LABEL: v_canonicalize_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX11TRUE16-NEXT: v_max_f32_e32 v0, v0, v0
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -32225,26 +31971,15 @@ define bfloat @v_canonicalize_bf16(bfloat %a) {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250TRUE16-LABEL: v_canonicalize_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_max_num_f32_e32 v0, v1, v1
-; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250FAKE16-LABEL: v_canonicalize_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_max_num_f32_e32 v0, v0, v0
-; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_canonicalize_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_max_num_f32_e32 v0, v0, v0
+; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = call bfloat @llvm.canonicalize.bf16(bfloat %a)
ret bfloat %op
}
@@ -32360,50 +32095,25 @@ define i1 @v_fcmp_oeq_bf16(bfloat %a, bfloat %b) {
; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_fcmp_oeq_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, v1, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fcmp_oeq_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, v0, v1
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250TRUE16-LABEL: v_fcmp_oeq_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX1250TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, v1, v2
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX11-LABEL: v_fcmp_oeq_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_eq_f32_e32 vcc_lo, v0, v1
+; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250FAKE16-LABEL: v_fcmp_oeq_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, v0, v1
-; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_fcmp_oeq_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmp_eq_f32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = fcmp oeq bfloat %a, %b
ret i1 %op
}
@@ -32464,50 +32174,25 @@ define i1 @v_fcmp_ogt_bf16(bfloat %a, bfloat %b) {
; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_fcmp_ogt_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fcmp_ogt_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v1
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250TRUE16-LABEL: v_fcmp_ogt_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX1250TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v2
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX11-LABEL: v_fcmp_ogt_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v1
+; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250FAKE16-LABEL: v_fcmp_ogt_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v1
-; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_fcmp_ogt_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = fcmp ogt bfloat %a, %b
ret i1 %op
}
@@ -32568,50 +32253,25 @@ define i1 @v_fcmp_oge_bf16(bfloat %a, bfloat %b) {
; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_fcmp_oge_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11TRUE16-NEXT: v_cmp_ge_f32_e32 vcc_lo, v1, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fcmp_oge_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cmp_ge_f32_e32 vcc_lo, v0, v1
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250TRUE16-LABEL: v_fcmp_oge_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX1250TRUE16-NEXT: v_cmp_ge_f32_e32 vcc_lo, v1, v2
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX11-LABEL: v_fcmp_oge_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_ge_f32_e32 vcc_lo, v0, v1
+; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250FAKE16-LABEL: v_fcmp_oge_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_cmp_ge_f32_e32 vcc_lo, v0, v1
-; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_fcmp_oge_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmp_ge_f32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = fcmp oge bfloat %a, %b
ret i1 %op
}
@@ -32672,50 +32332,25 @@ define i1 @v_fcmp_olt_bf16(bfloat %a, bfloat %b) {
; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_fcmp_olt_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fcmp_olt_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v1
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250TRUE16-LABEL: v_fcmp_olt_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX1250TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v2
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX11-LABEL: v_fcmp_olt_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v1
+; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250FAKE16-LABEL: v_fcmp_olt_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v1
-; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_fcmp_olt_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = fcmp olt bfloat %a, %b
ret i1 %op
}
@@ -32776,50 +32411,25 @@ define i1 @v_fcmp_ole_bf16(bfloat %a, bfloat %b) {
; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_fcmp_ole_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11TRUE16-NEXT: v_cmp_le_f32_e32 vcc_lo, v1, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fcmp_ole_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cmp_le_f32_e32 vcc_lo, v0, v1
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250TRUE16-LABEL: v_fcmp_ole_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX1250TRUE16-NEXT: v_cmp_le_f32_e32 vcc_lo, v1, v2
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX11-LABEL: v_fcmp_ole_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_le_f32_e32 vcc_lo, v0, v1
+; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250FAKE16-LABEL: v_fcmp_ole_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_cmp_le_f32_e32 vcc_lo, v0, v1
-; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_fcmp_ole_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmp_le_f32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = fcmp ole bfloat %a, %b
ret i1 %op
}
@@ -32880,50 +32490,25 @@ define i1 @v_fcmp_one_bf16(bfloat %a, bfloat %b) {
; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_fcmp_one_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11TRUE16-NEXT: v_cmp_lg_f32_e32 vcc_lo, v1, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fcmp_one_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cmp_lg_f32_e32 vcc_lo, v0, v1
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250TRUE16-LABEL: v_fcmp_one_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX1250TRUE16-NEXT: v_cmp_lg_f32_e32 vcc_lo, v1, v2
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX11-LABEL: v_fcmp_one_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_lg_f32_e32 vcc_lo, v0, v1
+; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250FAKE16-LABEL: v_fcmp_one_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_cmp_lg_f32_e32 vcc_lo, v0, v1
-; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_fcmp_one_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmp_lg_f32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = fcmp one bfloat %a, %b
ret i1 %op
}
@@ -32984,50 +32569,25 @@ define i1 @v_fcmp_uno_bf16(bfloat %a, bfloat %b) {
; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_fcmp_uno_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fcmp_uno_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v1
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250TRUE16-LABEL: v_fcmp_uno_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX1250TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v2
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX11-LABEL: v_fcmp_uno_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v1
+; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250FAKE16-LABEL: v_fcmp_uno_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v1
-; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_fcmp_uno_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = fcmp uno bfloat %a, %b
ret i1 %op
}
@@ -33088,50 +32648,25 @@ define i1 @v_fcmp_ueq_bf16(bfloat %a, bfloat %b) {
; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_fcmp_ueq_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11TRUE16-NEXT: v_cmp_nlg_f32_e32 vcc_lo, v1, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fcmp_ueq_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cmp_nlg_f32_e32 vcc_lo, v0, v1
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250TRUE16-LABEL: v_fcmp_ueq_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX1250TRUE16-NEXT: v_cmp_nlg_f32_e32 vcc_lo, v1, v2
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX11-LABEL: v_fcmp_ueq_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_nlg_f32_e32 vcc_lo, v0, v1
+; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250FAKE16-LABEL: v_fcmp_ueq_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_cmp_nlg_f32_e32 vcc_lo, v0, v1
-; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_fcmp_ueq_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmp_nlg_f32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = fcmp ueq bfloat %a, %b
ret i1 %op
}
@@ -33192,50 +32727,25 @@ define i1 @v_fcmp_ugt_bf16(bfloat %a, bfloat %b) {
; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_fcmp_ugt_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11TRUE16-NEXT: v_cmp_nle_f32_e32 vcc_lo, v1, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fcmp_ugt_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cmp_nle_f32_e32 vcc_lo, v0, v1
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250TRUE16-LABEL: v_fcmp_ugt_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX1250TRUE16-NEXT: v_cmp_nle_f32_e32 vcc_lo, v1, v2
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX11-LABEL: v_fcmp_ugt_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_nle_f32_e32 vcc_lo, v0, v1
+; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250FAKE16-LABEL: v_fcmp_ugt_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_cmp_nle_f32_e32 vcc_lo, v0, v1
-; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_fcmp_ugt_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmp_nle_f32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = fcmp ugt bfloat %a, %b
ret i1 %op
}
@@ -33296,50 +32806,25 @@ define i1 @v_fcmp_uge_bf16(bfloat %a, bfloat %b) {
; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_fcmp_uge_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, v1, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fcmp_uge_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, v0, v1
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250TRUE16-LABEL: v_fcmp_uge_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX1250TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, v1, v2
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX11-LABEL: v_fcmp_uge_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_nlt_f32_e32 vcc_lo, v0, v1
+; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250FAKE16-LABEL: v_fcmp_uge_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, v0, v1
-; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_fcmp_uge_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmp_nlt_f32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = fcmp uge bfloat %a, %b
ret i1 %op
}
@@ -33400,50 +32885,25 @@ define i1 @v_fcmp_ult_bf16(bfloat %a, bfloat %b) {
; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_fcmp_ult_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11TRUE16-NEXT: v_cmp_nge_f32_e32 vcc_lo, v1, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fcmp_ult_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cmp_nge_f32_e32 vcc_lo, v0, v1
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250TRUE16-LABEL: v_fcmp_ult_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX1250TRUE16-NEXT: v_cmp_nge_f32_e32 vcc_lo, v1, v2
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX11-LABEL: v_fcmp_ult_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_nge_f32_e32 vcc_lo, v0, v1
+; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250FAKE16-LABEL: v_fcmp_ult_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_cmp_nge_f32_e32 vcc_lo, v0, v1
-; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_fcmp_ult_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmp_nge_f32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = fcmp ult bfloat %a, %b
ret i1 %op
}
@@ -33504,50 +32964,25 @@ define i1 @v_fcmp_ule_bf16(bfloat %a, bfloat %b) {
; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_fcmp_ule_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, v1, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fcmp_ule_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, v0, v1
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250TRUE16-LABEL: v_fcmp_ule_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX1250TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, v1, v2
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX11-LABEL: v_fcmp_ule_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_ngt_f32_e32 vcc_lo, v0, v1
+; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250FAKE16-LABEL: v_fcmp_ule_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, v0, v1
-; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_fcmp_ule_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmp_ngt_f32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = fcmp ule bfloat %a, %b
ret i1 %op
}
@@ -33608,50 +33043,25 @@ define i1 @v_fcmp_une_bf16(bfloat %a, bfloat %b) {
; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_fcmp_une_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11TRUE16-NEXT: v_cmp_neq_f32_e32 vcc_lo, v1, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fcmp_une_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cmp_neq_f32_e32 vcc_lo, v0, v1
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250TRUE16-LABEL: v_fcmp_une_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX1250TRUE16-NEXT: v_cmp_neq_f32_e32 vcc_lo, v1, v2
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX11-LABEL: v_fcmp_une_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_neq_f32_e32 vcc_lo, v0, v1
+; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250FAKE16-LABEL: v_fcmp_une_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_cmp_neq_f32_e32 vcc_lo, v0, v1
-; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_fcmp_une_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cmp_neq_f32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = fcmp une bfloat %a, %b
ret i1 %op
}
@@ -33739,41 +33149,22 @@ define i16 @v_fptosi_bf16_to_i16(bfloat %x) {
; GFX10-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_fptosi_bf16_to_i16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v1
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fptosi_bf16_to_i16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cvt_i32_f32_e32 v0, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250TRUE16-LABEL: v_fptosi_bf16_to_i16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v1
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX11-LABEL: v_fptosi_bf16_to_i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250FAKE16-LABEL: v_fptosi_bf16_to_i16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_cvt_i32_f32_e32 v0, v0
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_fptosi_bf16_to_i16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = fptosi bfloat %x to i16
ret i16 %op
}
@@ -34137,41 +33528,22 @@ define i32 @v_fptosi_bf16_to_i32(bfloat %x) {
; GFX10-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_fptosi_bf16_to_i32:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v1
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fptosi_bf16_to_i32:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cvt_i32_f32_e32 v0, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250TRUE16-LABEL: v_fptosi_bf16_to_i32:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v1
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX11-LABEL: v_fptosi_bf16_to_i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250FAKE16-LABEL: v_fptosi_bf16_to_i32:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_cvt_i32_f32_e32 v0, v0
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_fptosi_bf16_to_i32:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = fptosi bfloat %x to i32
ret i32 %op
}
@@ -34559,93 +33931,48 @@ define i64 @v_fptosi_bf16_to_i64(bfloat %x) {
; GFX10-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_fptosi_bf16_to_i64:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_trunc_f32_e32 v0, v1
-; GFX11TRUE16-NEXT: v_mul_f32_e64 v1, 0x2f800000, |v0|
-; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v3, 31, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_floor_f32_e32 v1, v1
-; GFX11TRUE16-NEXT: v_fma_f32 v2, 0xcf800000, v1, |v0|
-; GFX11TRUE16-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cvt_u32_f32_e32 v0, v2
-; GFX11TRUE16-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_xor_b32_e32 v0, v0, v3
-; GFX11TRUE16-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_sub_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fptosi_bf16_to_i64:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_trunc_f32_e32 v0, v0
-; GFX11FAKE16-NEXT: v_mul_f32_e64 v1, 0x2f800000, |v0|
-; GFX11FAKE16-NEXT: v_ashrrev_i32_e32 v3, 31, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_floor_f32_e32 v1, v1
-; GFX11FAKE16-NEXT: v_fma_f32 v2, 0xcf800000, v1, |v0|
-; GFX11FAKE16-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT: v_cvt_u32_f32_e32 v0, v2
-; GFX11FAKE16-NEXT: v_xor_b32_e32 v1, v1, v3
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_xor_b32_e32 v0, v0, v3
-; GFX11FAKE16-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v3
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_sub_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250TRUE16-LABEL: v_fptosi_bf16_to_i64:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_trunc_f32_e32 v0, v1
-; GFX1250TRUE16-NEXT: v_mul_f32_e64 v1, 0x2f800000, |v0|
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_floor_f32_e32 v1, v1
-; GFX1250TRUE16-NEXT: v_fma_f32 v2, 0xcf800000, v1, |v0|
-; GFX1250TRUE16-NEXT: v_ashrrev_i32_e32 v0, 31, v0
-; GFX1250TRUE16-NEXT: v_cvt_u32_f32_e32 v3, v1
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250TRUE16-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX1250TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_bitop2_b32 v3, v3, v0 bitop3:0x14
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_xor_b32_e32 v2, v2, v0
-; GFX1250TRUE16-NEXT: v_sub_nc_u64_e32 v[0:1], v[2:3], v[0:1]
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX11-LABEL: v_fptosi_bf16_to_i64:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_trunc_f32_e32 v0, v0
+; GFX11-NEXT: v_mul_f32_e64 v1, 0x2f800000, |v0|
+; GFX11-NEXT: v_ashrrev_i32_e32 v3, 31, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_floor_f32_e32 v1, v1
+; GFX11-NEXT: v_fma_f32 v2, 0xcf800000, v1, |v0|
+; GFX11-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v2
+; GFX11-NEXT: v_xor_b32_e32 v1, v1, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_xor_b32_e32 v0, v0, v3
+; GFX11-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_sub_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250FAKE16-LABEL: v_fptosi_bf16_to_i64:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_trunc_f32_e32 v0, v0
-; GFX1250FAKE16-NEXT: v_mul_f32_e64 v1, 0x2f800000, |v0|
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_floor_f32_e32 v1, v1
-; GFX1250FAKE16-NEXT: v_fma_f32 v2, 0xcf800000, v1, |v0|
-; GFX1250FAKE16-NEXT: v_ashrrev_i32_e32 v0, 31, v0
-; GFX1250FAKE16-NEXT: v_cvt_u32_f32_e32 v3, v1
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250FAKE16-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX1250FAKE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_bitop2_b32 v3, v3, v0 bitop3:0x14
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_xor_b32_e32 v2, v2, v0
-; GFX1250FAKE16-NEXT: v_sub_nc_u64_e32 v[0:1], v[2:3], v[0:1]
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_fptosi_bf16_to_i64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_trunc_f32_e32 v0, v0
+; GFX1250-NEXT: v_mul_f32_e64 v1, 0x2f800000, |v0|
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_floor_f32_e32 v1, v1
+; GFX1250-NEXT: v_fma_f32 v2, 0xcf800000, v1, |v0|
+; GFX1250-NEXT: v_ashrrev_i32_e32 v0, 31, v0
+; GFX1250-NEXT: v_cvt_u32_f32_e32 v3, v1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX1250-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_bitop2_b32 v3, v3, v0 bitop3:0x14
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_xor_b32_e32 v2, v2, v0
+; GFX1250-NEXT: v_sub_nc_u64_e32 v[0:1], v[2:3], v[0:1]
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = fptosi bfloat %x to i64
ret i64 %op
}
@@ -42375,7 +41702,7 @@ define amdgpu_ps i32 @s_select_bf16(bfloat inreg %a, bfloat inreg %b, i32 %c) {
;
; GFX1250TRUE16-LABEL: s_select_bf16:
; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, s0
; GFX1250TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1250TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
@@ -42386,7 +41713,7 @@ define amdgpu_ps i32 @s_select_bf16(bfloat inreg %a, bfloat inreg %b, i32 %c) {
;
; GFX1250FAKE16-LABEL: s_select_bf16:
; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250FAKE16-NEXT: v_mov_b32_e32 v1, s0
; GFX1250FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -42531,7 +41858,7 @@ define amdgpu_ps i32 @s_select_v2bf16(<2 x bfloat> inreg %a, <2 x bfloat> inreg
;
; GFX1250TRUE16-LABEL: s_select_v2bf16:
; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250TRUE16-NEXT: s_lshr_b32 s2, s0, 16
; GFX1250TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, s2
@@ -42546,7 +41873,7 @@ define amdgpu_ps i32 @s_select_v2bf16(<2 x bfloat> inreg %a, <2 x bfloat> inreg
;
; GFX1250FAKE16-LABEL: s_select_v2bf16:
; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250FAKE16-NEXT: s_lshr_b32 s2, s0, 16
; GFX1250FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1250FAKE16-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s0
@@ -42693,7 +42020,7 @@ define amdgpu_ps i32 @s_vselect_v2bf16(<2 x bfloat> inreg %a, <2 x bfloat> inreg
;
; GFX1250TRUE16-LABEL: s_vselect_v2bf16:
; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250TRUE16-NEXT: s_lshr_b32 s3, s0, 16
; GFX1250TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1250TRUE16-NEXT: v_cmp_eq_u32_e64 s2, 0, v1
@@ -42709,7 +42036,7 @@ define amdgpu_ps i32 @s_vselect_v2bf16(<2 x bfloat> inreg %a, <2 x bfloat> inreg
;
; GFX1250FAKE16-LABEL: s_vselect_v2bf16:
; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250FAKE16-NEXT: s_lshr_b32 s2, s0, 16
; GFX1250FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX1250FAKE16-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s0
@@ -43495,7 +42822,7 @@ define amdgpu_ps <2 x i32> @s_select_v3bf16(<3 x bfloat> inreg %a, <3 x bfloat>
;
; GFX1250-LABEL: s_select_v3bf16:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v2, s1
; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
@@ -43612,7 +42939,7 @@ define amdgpu_ps <2 x i32> @s_select_v4bf16(<4 x bfloat> inreg %a, <4 x bfloat>
;
; GFX1250-LABEL: s_select_v4bf16:
; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s0
; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
@@ -43845,7 +43172,7 @@ define amdgpu_ps <2 x i32> @s_vselect_v4bf16(<4 x bfloat> inreg %a, <4 x bfloat>
;
; GFX1250TRUE16-LABEL: s_vselect_v4bf16:
; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250TRUE16-NEXT: s_lshr_b32 s7, s1, 16
; GFX1250TRUE16-NEXT: s_lshr_b32 s9, s0, 16
; GFX1250TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
@@ -43869,7 +43196,7 @@ define amdgpu_ps <2 x i32> @s_vselect_v4bf16(<4 x bfloat> inreg %a, <4 x bfloat>
;
; GFX1250FAKE16-LABEL: s_vselect_v4bf16:
; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
+; GFX1250FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250FAKE16-NEXT: s_lshr_b32 s4, s1, 16
; GFX1250FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v3
; GFX1250FAKE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s1
@@ -47179,21 +46506,18 @@ define bfloat @v_fma_bf16(bfloat %a, bfloat %b, bfloat %c) {
; GFX11TRUE16-LABEL: v_fma_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v3, v1, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v0, v3, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v3
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v3, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v2, v0, v1
+; GFX11TRUE16-NEXT: v_bfe_u32 v0, v2, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v2
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v2, 0x7fff
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -52151,21 +51475,18 @@ define bfloat @v_fmuladd_bf16(bfloat %a, bfloat %b, bfloat %c) {
; GFX11TRUE16-LABEL: v_fmuladd_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v3, v1, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v0, v3, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v3
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v3, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v2, v0, v1
+; GFX11TRUE16-NEXT: v_bfe_u32 v0, v2, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v2
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v2, 0x7fff
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index 029512631b367..b5844049fd287 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -4990,56 +4990,55 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, s4
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s4
; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v5, s[0:3], null offen
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
; GFX12-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v0, v0, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v2
; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB16_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
@@ -5147,54 +5146,53 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s4
; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v5, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v3
-; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v2
; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB16_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
@@ -5510,49 +5508,48 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s4
; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v3, s[0:3], null offen
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
; GFX12-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v1, v1, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v4.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4
; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
@@ -5664,47 +5661,46 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s4
; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v3, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, v1, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v4.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v4
-; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v4
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4
; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
@@ -6018,15 +6014,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
+; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v6
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, -4, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v11, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v9, v6
; GFX12-TRUE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
@@ -6040,38 +6036,39 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v10, s[4:7], null offen
+; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX12-TRUE16-NEXT: ; %bb.2:
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX12-TRUE16-NEXT: s_mov_b32 s1, 0
; GFX12-TRUE16-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB18_4 Depth 2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.h, v5.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX12-TRUE16-NEXT: s_mov_b32 s2, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v6, v6, v8
-; GFX12-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v6
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v4, v4, v10
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v12, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v6
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-TRUE16-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
@@ -6086,14 +6083,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_4
; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v4
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6101,7 +6098,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_3
; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -6275,16 +6272,16 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, -4, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v11, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-TRUE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
@@ -6296,38 +6293,39 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v10, s[4:7], 0 offen
+; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX11-TRUE16-NEXT: ; %bb.2:
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB18_3: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-TRUE16-NEXT: ; Child Loop BB18_4 Depth 2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, v6, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v12, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, v4, v10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-TRUE16-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
@@ -6341,14 +6339,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], 0 offen glc
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v4
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
@@ -6358,7 +6356,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index 1b957444869e7..3d85bf7019426 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -4131,56 +4131,55 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, s4
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s4
; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v5, s[0:3], null offen
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
; GFX12-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v1, v1, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v0, v0, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v2
; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB13_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
@@ -4288,54 +4287,53 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s4
; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v5, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v1, v1, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v0, v0, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v3
-; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v2
; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB13_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
@@ -4653,49 +4651,48 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s4
; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v3, s[0:3], null offen
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
; GFX12-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v1, v1, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v4.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v0, v0, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4
; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
@@ -4807,47 +4804,46 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s4
; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v3, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v1, v1, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v4.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v4
-; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v4
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4
; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
@@ -5163,15 +5159,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
+; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v6
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, -4, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v11, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v9, v6
; GFX12-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
@@ -5185,38 +5181,39 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v10, s[4:7], null offen
+; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX12-TRUE16-NEXT: ; %bb.2:
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX12-TRUE16-NEXT: s_mov_b32 s1, 0
; GFX12-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.h, v5.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX12-TRUE16-NEXT: s_mov_b32 s2, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v8
-; GFX12-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v6
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v4, v4, v10
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v12, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v6
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
@@ -5231,14 +5228,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v4
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -5246,7 +5243,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -5420,16 +5417,16 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, -4, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v11, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
@@ -5441,38 +5438,39 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v10, s[4:7], 0 offen
+; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-TRUE16-NEXT: ; %bb.2:
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-TRUE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v12, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v4, v4, v10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
@@ -5486,14 +5484,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], 0 offen glc
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v4
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
@@ -5503,7 +5501,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index da140ac4bf59c..3f15d35320573 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -4131,56 +4131,55 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, s4
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s4
; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v5, s[0:3], null offen
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
; GFX12-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v1, v1, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v0, v0, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v2
; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB13_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
@@ -4288,54 +4287,53 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s4
; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v5, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v1, v1, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v0, v0, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v3
-; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v2
; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB13_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
@@ -4653,49 +4651,48 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s4
; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v3, s[0:3], null offen
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
; GFX12-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v1, v1, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v4.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v0, v0, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4
; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
@@ -4807,47 +4804,46 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s4
; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v3, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v1, v1, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v4.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v4
-; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v4
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4
; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
@@ -5163,15 +5159,15 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
+; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v6
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, -4, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v11, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v9, v6
; GFX12-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
@@ -5185,38 +5181,39 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v10, s[4:7], null offen
+; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v8, s[4:7], null offen
; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX12-TRUE16-NEXT: ; %bb.2:
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX12-TRUE16-NEXT: s_mov_b32 s1, 0
; GFX12-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.h, v5.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX12-TRUE16-NEXT: s_mov_b32 s2, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v8
-; GFX12-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v6
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v4, v4, v10
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v12, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v6
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
@@ -5231,14 +5228,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v4
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -5246,7 +5243,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_3
; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -5420,16 +5417,16 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, -4, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v11, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v1
@@ -5441,38 +5438,39 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v10, s[4:7], 0 offen
+; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[4:7], 0 offen
; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_1
; GFX11-TRUE16-NEXT: ; %bb.2:
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-TRUE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX11-TRUE16-NEXT: s_mov_b32 s2, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v12, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v4, v4, v10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
@@ -5486,14 +5484,14 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[4:7], 0 offen glc
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[4:7], 0 offen glc
; GFX11-TRUE16-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v4
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: s_or_b32 s1, vcc_lo, s1
@@ -5503,7 +5501,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
index 76583e806b805..3aa89a7210ca0 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
@@ -2782,20 +2782,20 @@ define bfloat @fmul_select_bf16_test1(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x3f80
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0x4000, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x4000, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -2823,20 +2823,20 @@ define bfloat @fmul_select_bf16_test1(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x3f80
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0x4000, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x4000, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -2915,20 +2915,20 @@ define bfloat @fmul_select_bf16_test2(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x3f80
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0x3f00, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x3f00, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -2956,20 +2956,20 @@ define bfloat @fmul_select_bf16_test2(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x3f80
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0x3f00, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x3f00, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3073,31 +3073,32 @@ define <2 x bfloat> @fmul_select_v2bf16_test3(<2 x bfloat> %x, <2 x i32> %bool.a
; GFX11-SDAG-TRUE16-LABEL: fmul_select_v2bf16_test3:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0x3f80
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v1, v3
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v2.l, 0x4000, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, 0x4000, s0
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0x3f80
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v3
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, 0x4000, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, 0x4000, s0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v2, v4, v3
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v3
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX11-SDAG-TRUE16-NEXT: v_dual_mul_f32 v0, v0, v2 :: v_dual_mul_f32 v1, v3, v1
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3136,31 +3137,32 @@ define <2 x bfloat> @fmul_select_v2bf16_test3(<2 x bfloat> %x, <2 x i32> %bool.a
; GFX11-GISEL-TRUE16-LABEL: fmul_select_v2bf16_test3:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0x3f80
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v1, v3
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v3.h, v2.l, 0x4000, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, 0x4000, s0
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0x3f80
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v3
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, 0x4000, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, 0x4000, s0
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v2, v4, v3
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v3
-; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mul_f32 v0, v0, v2 :: v_dual_mul_f32 v1, v3, v1
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3276,31 +3278,32 @@ define <2 x bfloat> @fmul_select_v2bf16_test4(<2 x bfloat> %x, <2 x i32> %bool.a
; GFX11-SDAG-TRUE16-LABEL: fmul_select_v2bf16_test4:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0x3f80
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v1, v3
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v2.l, 0x3f00, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, 0x3f00, s0
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0x3f80
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v3
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, 0x3f00, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, 0x3f00, s0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v2, v4, v3
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v3
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX11-SDAG-TRUE16-NEXT: v_dual_mul_f32 v0, v0, v2 :: v_dual_mul_f32 v1, v3, v1
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3339,31 +3342,32 @@ define <2 x bfloat> @fmul_select_v2bf16_test4(<2 x bfloat> %x, <2 x i32> %bool.a
; GFX11-GISEL-TRUE16-LABEL: fmul_select_v2bf16_test4:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0x3f80
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v1, v3
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v3.h, v2.l, 0x3f00, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, 0x3f00, s0
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0x3f80
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v3
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v4
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, 0x3f00, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, 0x3f00, s0
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v2, v4, v3
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v3
-; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX11-GISEL-TRUE16-NEXT: v_dual_mul_f32 v0, v0, v2 :: v_dual_mul_f32 v1, v3, v1
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3455,20 +3459,20 @@ define bfloat @fmul_select_bf16_test5(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4100
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0x4000, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x4000, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3496,20 +3500,20 @@ define bfloat @fmul_select_bf16_test5(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4100
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0x4000, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x4000, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3590,20 +3594,20 @@ define bfloat @fmul_select_bf16_test6(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4040
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0xc100, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0xc100, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3631,20 +3635,20 @@ define bfloat @fmul_select_bf16_test6(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4040
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0xc100, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0xc100, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3724,20 +3728,20 @@ define bfloat @fmul_select_bf16_test7(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xc080
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0x4100, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x4100, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3765,20 +3769,20 @@ define bfloat @fmul_select_bf16_test7(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xc080
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0x4100, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x4100, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3856,13 +3860,12 @@ define bfloat @fmul_select_bf16_test8(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test8:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, 0, 0x8000, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v3, v0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 0x8000, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
@@ -3895,13 +3898,12 @@ define bfloat @fmul_select_bf16_test8(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-GISEL-TRUE16-LABEL: fmul_select_bf16_test8:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v0.h, 0, 0x8000, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v3, v0
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 0x8000, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
@@ -3988,20 +3990,20 @@ define bfloat @fmul_select_bf16_test9(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xc200
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0xc180, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0xc180, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4029,20 +4031,20 @@ define bfloat @fmul_select_bf16_test9(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xc200
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0xc180, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0xc180, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4123,20 +4125,20 @@ define bfloat @fmul_select_bf16_test10_sel_log2val_pos65_pos56(bfloat %x, i32 %b
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xdb80
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0xe000, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0xe000, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4164,20 +4166,20 @@ define bfloat @fmul_select_bf16_test10_sel_log2val_pos65_pos56(bfloat %x, i32 %b
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xdb80
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0xe000, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0xe000, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4258,20 +4260,20 @@ define bfloat @fmul_select_bf16_test11_sel_log2val_neg22_pos25(bfloat %x, i32 %b
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4c00
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0x3480, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x3480, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4299,20 +4301,20 @@ define bfloat @fmul_select_bf16_test11_sel_log2val_neg22_pos25(bfloat %x, i32 %b
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4c00
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
; GFX11-GISEL-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, 0x3480, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v4, v0
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x3480, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll b/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll
index d34334882e9bf..46ec2dd9ceeab 100644
--- a/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll
+++ b/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll
@@ -77,20 +77,11 @@ define i32 @divergent_vec_0_i16(i16 %a) {
; GFX906-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: divergent_vec_0_i16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v1
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: divergent_vec_0_i16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: divergent_vec_0_i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%tmp = insertelement <2 x i16> poison, i16 0, i32 0
%vec = insertelement <2 x i16> %tmp, i16 %a, i32 1
%val = bitcast <2 x i16> %vec to i32
diff --git a/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll b/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll
index c46fcde739b1c..9018d71e7a59c 100644
--- a/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll
@@ -615,35 +615,35 @@ define amdgpu_kernel void @v_fabs_fold_self_v2bf16(ptr addrspace(1) %out, ptr ad
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[2:3]
+; GFX11-TRUE16-NEXT: global_load_b32 v2, v0, s[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_mul_f32 v2, v1, v2 :: v_dual_and_b32 v3, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0x7fff, v2.l
+; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0x7fff, v2.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_dual_mul_f32 v0, v0, v3 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mul_f32_e32 v1, v1, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_cndmask_b32 v1, v3, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
-; GFX11-TRUE16-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-TRUE16-NEXT: global_store_b32 v2, v1, s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
;
; GFX11-FAKE16-LABEL: v_fabs_fold_self_v2bf16:
@@ -798,36 +798,35 @@ define amdgpu_kernel void @v_fabs_fold_v2bf16(ptr addrspace(1) %out, ptr addrspa
; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-TRUE16-NEXT: s_load_b32 s4, s[4:5], 0x10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[2:3]
; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s4, 16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mul_f32_e32 v2, s2, v1
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.h
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0x7fff, v0.l
+; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0x7fff, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_dual_mul_f32 v0, s2, v0 :: v_dual_lshlrev_b32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_and_b32 s2, s4, 0xffff0000
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mul_f32_e32 v0, s2, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_mul_f32_e32 v1, s2, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
-; GFX11-TRUE16-NEXT: global_store_b32 v2, v0, s[0:1]
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_cndmask_b32 v1, v3, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-TRUE16-NEXT: global_store_b32 v2, v1, s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
;
; GFX11-FAKE16-LABEL: v_fabs_fold_v2bf16:
@@ -973,36 +972,34 @@ define amdgpu_kernel void @v_extract_fabs_fold_v2bf16(ptr addrspace(1) %in) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[0:1]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0x7fff, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mul_f32_e32 v2, 4.0, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 2.0, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0x7fff, v0.l
+; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0x7fff, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_dual_mul_f32 v0, 4.0, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 2.0, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: global_store_d16_hi_b16 v[0:1], v1, off dlc
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
; GFX11-TRUE16-NEXT: global_store_d16_hi_b16 v[0:1], v0, off dlc
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_store_d16_hi_b16 v[0:1], v1, off dlc
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: s_endpgm
;
; GFX11-FAKE16-LABEL: v_extract_fabs_fold_v2bf16:
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.bf16.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.bf16.ll
index 9c0a7a16f4e50..0896a540d09d0 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize-elimination.bf16.ll
@@ -32,10 +32,10 @@ define bfloat @test_canonicalize_amdgcn_tanh_bf16(bfloat %a) {
; REAL16: ; %bb.0:
; REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; REAL16-NEXT: s_wait_kmcnt 0x0
-; REAL16-NEXT: v_tanh_bf16_e32 v0.h, v0.l
+; REAL16-NEXT: v_tanh_bf16_e32 v0.l, v0.l
; REAL16-NEXT: v_nop
-; REAL16-NEXT: v_mov_b16_e32 v0.l, 0
-; REAL16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instid1(VALU_DEP_1)
+; REAL16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; REAL16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; REAL16-NEXT: v_max_num_f32_e32 v0, v0, v0
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; REAL16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
index 5805552c0132c..b8dc2912072ba 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
@@ -81,29 +81,15 @@ define amdgpu_kernel void @s_test_canonicalize_var_bf16(ptr addrspace(1) %out, i
}
define <2 x bfloat> @v_test_canonicalize_build_vector_v2bf16(bfloat %lo, bfloat %hi) #1 {
-; FAKE16-LABEL: v_test_canonicalize_build_vector_v2bf16:
-; FAKE16: ; %bb.0:
-; FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; FAKE16-NEXT: s_wait_kmcnt 0x0
-; FAKE16-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
-; FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; FAKE16-NEXT: v_dual_max_num_f32 v1, v1, v1 :: v_dual_max_num_f32 v0, v0, v0
-; FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1
-; FAKE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; REAL16-LABEL: v_test_canonicalize_build_vector_v2bf16:
-; REAL16: ; %bb.0:
-; REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
-; REAL16-NEXT: s_wait_kmcnt 0x0
-; REAL16-NEXT: v_mov_b16_e32 v2.l, 0
-; REAL16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; REAL16-NEXT: v_max_num_f32_e32 v1, v2, v2
-; REAL16-NEXT: v_mov_b16_e32 v2.h, v0.l
-; REAL16-NEXT: v_max_num_f32_e32 v0, v2, v2
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; REAL16-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1
-; REAL16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_test_canonicalize_build_vector_v2bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_dual_max_num_f32 v1, v1, v1 :: v_dual_max_num_f32 v0, v0, v0
+; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%ins0 = insertelement <2 x bfloat> poison, bfloat %lo, i32 0
%ins1 = insertelement <2 x bfloat> %ins0, bfloat %hi, i32 1
%canonicalized = call <2 x bfloat> @llvm.canonicalize.v2bf16(<2 x bfloat> %ins1)
@@ -757,18 +743,18 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_v2bf16(ptr addrspace(1)
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; REAL16-NEXT: v_mov_b16_e32 v1.l, 0
+; REAL16-NEXT: v_mov_b32_e32 v2, 0
; REAL16-NEXT: s_wait_kmcnt 0x0
; REAL16-NEXT: global_load_b32 v0, v0, s[0:1] scale_offset
; REAL16-NEXT: s_wait_loadcnt 0x0
-; REAL16-NEXT: v_and_b16 v1.h, 0x7fff, v0.l
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; REAL16-NEXT: v_max_num_f32_e32 v2, v1, v1
-; REAL16-NEXT: v_and_b16 v1.h, 0x7fff, v0.h
-; REAL16-NEXT: v_dual_max_num_f32 v0, v1, v1 :: v_dual_mov_b32 v1, 0
+; REAL16-NEXT: v_and_b16 v0.l, 0x7fff, v0.l
+; REAL16-NEXT: v_and_b16 v1.l, 0x7fff, v0.h
+; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; REAL16-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; REAL16-NEXT: v_dual_max_num_f32 v0, v0, v0 :: v_dual_max_num_f32 v1, v1, v1
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; REAL16-NEXT: v_cvt_pk_bf16_f32 v0, v2, v0
-; REAL16-NEXT: global_store_b32 v1, v0, s[0:1]
+; REAL16-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1
+; REAL16-NEXT: global_store_b32 v2, v0, s[0:1]
; REAL16-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %out, i32 %tid
@@ -807,18 +793,18 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_v2bf16(ptr addrspac
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; REAL16-NEXT: v_mov_b16_e32 v1.l, 0
+; REAL16-NEXT: v_mov_b32_e32 v2, 0
; REAL16-NEXT: s_wait_kmcnt 0x0
; REAL16-NEXT: global_load_b32 v0, v0, s[0:1] scale_offset
; REAL16-NEXT: s_wait_loadcnt 0x0
-; REAL16-NEXT: v_or_b16 v1.h, 0x8000, v0.l
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; REAL16-NEXT: v_max_num_f32_e32 v2, v1, v1
-; REAL16-NEXT: v_or_b16 v1.h, 0x8000, v0.h
-; REAL16-NEXT: v_dual_max_num_f32 v0, v1, v1 :: v_dual_mov_b32 v1, 0
+; REAL16-NEXT: v_or_b16 v0.l, 0x8000, v0.l
+; REAL16-NEXT: v_or_b16 v1.l, 0x8000, v0.h
+; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; REAL16-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; REAL16-NEXT: v_dual_max_num_f32 v0, v0, v0 :: v_dual_max_num_f32 v1, v1, v1
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; REAL16-NEXT: v_cvt_pk_bf16_f32 v0, v2, v0
-; REAL16-NEXT: global_store_b32 v1, v0, s[0:1]
+; REAL16-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1
+; REAL16-NEXT: global_store_b32 v2, v0, s[0:1]
; REAL16-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %out, i32 %tid
@@ -858,18 +844,18 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_var_v2bf16(ptr addrspace(1)
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; REAL16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; REAL16-NEXT: v_mov_b16_e32 v1.l, 0
+; REAL16-NEXT: v_mov_b32_e32 v2, 0
; REAL16-NEXT: s_wait_kmcnt 0x0
; REAL16-NEXT: global_load_b32 v0, v0, s[0:1] scale_offset
; REAL16-NEXT: s_wait_loadcnt 0x0
-; REAL16-NEXT: v_xor_b16 v1.h, 0x8000, v0.l
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; REAL16-NEXT: v_max_num_f32_e32 v2, v1, v1
-; REAL16-NEXT: v_xor_b16 v1.h, 0x8000, v0.h
-; REAL16-NEXT: v_dual_max_num_f32 v0, v1, v1 :: v_dual_mov_b32 v1, 0
+; REAL16-NEXT: v_xor_b16 v0.l, 0x8000, v0.l
+; REAL16-NEXT: v_xor_b16 v1.l, 0x8000, v0.h
+; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; REAL16-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; REAL16-NEXT: v_dual_max_num_f32 v0, v0, v0 :: v_dual_max_num_f32 v1, v1, v1
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; REAL16-NEXT: v_cvt_pk_bf16_f32 v0, v2, v0
-; REAL16-NEXT: global_store_b32 v1, v0, s[0:1]
+; REAL16-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1
+; REAL16-NEXT: global_store_b32 v2, v0, s[0:1]
; REAL16-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr <2 x bfloat>, ptr addrspace(1) %out, i32 %tid
@@ -1209,10 +1195,9 @@ define <2 x bfloat> @v_test_canonicalize_reg_undef_v2bf16(bfloat %val) #1 {
; REAL16: ; %bb.0:
; REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; REAL16-NEXT: s_wait_kmcnt 0x0
-; REAL16-NEXT: v_mov_b16_e32 v1.l, 0
-; REAL16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; REAL16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; REAL16-NEXT: v_max_num_f32_e32 v0, v1, v1
+; REAL16-NEXT: v_max_num_f32_e32 v0, v0, v0
; REAL16-NEXT: v_cvt_pk_bf16_f32 v1, v0, s0
; REAL16-NEXT: v_mov_b16_e32 v0.h, 0x7fc0
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2)
@@ -1241,10 +1226,9 @@ define <2 x bfloat> @v_test_canonicalize_undef_reg_v2bf16(bfloat %val) #1 {
; REAL16: ; %bb.0:
; REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; REAL16-NEXT: s_wait_kmcnt 0x0
-; REAL16-NEXT: v_mov_b16_e32 v1.l, 0
-; REAL16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; REAL16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; REAL16-NEXT: v_max_num_f32_e32 v0, v1, v1
+; REAL16-NEXT: v_max_num_f32_e32 v0, v0, v0
; REAL16-NEXT: v_cvt_pk_bf16_f32 v1, v0, s0
; REAL16-NEXT: v_mov_b16_e32 v0.l, 0x7fc0
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2)
@@ -1321,10 +1305,9 @@ define <2 x bfloat> @v_test_canonicalize_reg_k_v2bf16(bfloat %val) #1 {
; REAL16: ; %bb.0:
; REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; REAL16-NEXT: s_wait_kmcnt 0x0
-; REAL16-NEXT: v_mov_b16_e32 v1.l, 0
-; REAL16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; REAL16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; REAL16-NEXT: v_max_num_f32_e32 v0, v1, v1
+; REAL16-NEXT: v_max_num_f32_e32 v0, v0, v0
; REAL16-NEXT: v_cvt_pk_bf16_f32 v1, v0, s0
; REAL16-NEXT: v_mov_b16_e32 v0.h, 0x4000
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2)
@@ -1354,10 +1337,9 @@ define <2 x bfloat> @v_test_canonicalize_k_reg_v2bf16(bfloat %val) #1 {
; REAL16: ; %bb.0:
; REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; REAL16-NEXT: s_wait_kmcnt 0x0
-; REAL16-NEXT: v_mov_b16_e32 v1.l, 0
-; REAL16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; REAL16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; REAL16-NEXT: v_max_num_f32_e32 v0, v1, v1
+; REAL16-NEXT: v_max_num_f32_e32 v0, v0, v0
; REAL16-NEXT: v_cvt_pk_bf16_f32 v1, v0, s0
; REAL16-NEXT: v_mov_b16_e32 v0.l, 0x4000
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2)
@@ -1403,10 +1385,9 @@ define <4 x bfloat> @v_test_canonicalize_reg_undef_undef_undef_v4bf16(bfloat %va
; REAL16: ; %bb.0:
; REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; REAL16-NEXT: s_wait_kmcnt 0x0
-; REAL16-NEXT: v_mov_b16_e32 v1.l, 0
-; REAL16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; REAL16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; REAL16-NEXT: v_max_num_f32_e32 v0, v1, v1
+; REAL16-NEXT: v_max_num_f32_e32 v0, v0, v0
; REAL16-NEXT: v_cvt_pk_bf16_f32 v1, v0, s0
; REAL16-NEXT: v_mov_b16_e32 v0.h, 0x7fc0
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2)
@@ -1419,31 +1400,16 @@ define <4 x bfloat> @v_test_canonicalize_reg_undef_undef_undef_v4bf16(bfloat %va
}
define <4 x bfloat> @v_test_canonicalize_reg_reg_undef_undef_v4bf16(bfloat %val0, bfloat %val1) #1 {
-; FAKE16-LABEL: v_test_canonicalize_reg_reg_undef_undef_v4bf16:
-; FAKE16: ; %bb.0:
-; FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; FAKE16-NEXT: s_wait_kmcnt 0x0
-; FAKE16-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
-; FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; FAKE16-NEXT: v_dual_max_num_f32 v1, v1, v1 :: v_dual_max_num_f32 v0, v0, v0
-; FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1
-; FAKE16-NEXT: v_mov_b32_e32 v1, 0x7fc07fc0
-; FAKE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; REAL16-LABEL: v_test_canonicalize_reg_reg_undef_undef_v4bf16:
-; REAL16: ; %bb.0:
-; REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
-; REAL16-NEXT: s_wait_kmcnt 0x0
-; REAL16-NEXT: v_mov_b16_e32 v2.l, 0
-; REAL16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; REAL16-NEXT: v_max_num_f32_e32 v1, v2, v2
-; REAL16-NEXT: v_mov_b16_e32 v2.h, v0.l
-; REAL16-NEXT: v_max_num_f32_e32 v0, v2, v2
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; REAL16-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1
-; REAL16-NEXT: v_mov_b32_e32 v1, 0x7fc07fc0
-; REAL16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_test_canonicalize_reg_reg_undef_undef_v4bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_dual_max_num_f32 v1, v1, v1 :: v_dual_max_num_f32 v0, v0, v0
+; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, v1
+; GFX1250-NEXT: v_mov_b32_e32 v1, 0x7fc07fc0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%vec0 = insertelement <4 x bfloat> poison, bfloat %val0, i32 0
%vec1 = insertelement <4 x bfloat> %vec0, bfloat %val1, i32 1
%canonicalized = call <4 x bfloat> @llvm.canonicalize.v4bf16(<4 x bfloat> %vec1)
@@ -1471,20 +1437,16 @@ define <4 x bfloat> @v_test_canonicalize_reg_undef_reg_reg_v4bf16(bfloat %val0,
; REAL16: ; %bb.0:
; REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; REAL16-NEXT: s_wait_kmcnt 0x0
-; REAL16-NEXT: v_mov_b16_e32 v3.l, 0
-; REAL16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; REAL16-NEXT: v_max_num_f32_e32 v0, v3, v3
-; REAL16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; REAL16-NEXT: v_max_num_f32_e32 v2, v3, v3
-; REAL16-NEXT: v_mov_b16_e32 v3.h, v1.l
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; REAL16-NEXT: v_cvt_pk_bf16_f32 v1, v0, s0
+; REAL16-NEXT: v_dual_lshlrev_b32 v0, 16, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; REAL16-NEXT: v_dual_max_num_f32 v0, v0, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; REAL16-NEXT: v_dual_max_num_f32 v2, v2, v2 :: v_dual_max_num_f32 v1, v1, v1
+; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; REAL16-NEXT: v_cvt_pk_bf16_f32 v3, v0, s0
; REAL16-NEXT: v_mov_b16_e32 v0.h, 0x7fc0
-; REAL16-NEXT: v_max_num_f32_e32 v3, v3, v3
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; REAL16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; REAL16-NEXT: v_cvt_pk_bf16_f32 v1, v3, v2
+; REAL16-NEXT: v_cvt_pk_bf16_f32 v1, v1, v2
+; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; REAL16-NEXT: v_mov_b16_e32 v0.l, v3.l
; REAL16-NEXT: s_set_pc_i64 s[30:31]
%vec0 = insertelement <4 x bfloat> poison, bfloat %val0, i32 0
%vec1 = insertelement <4 x bfloat> %vec0, bfloat %val1, i32 2
diff --git a/llvm/test/CodeGen/AMDGPU/fdiv.bf16.ll b/llvm/test/CodeGen/AMDGPU/fdiv.bf16.ll
index 00cde422a2297..6a38290e04a00 100644
--- a/llvm/test/CodeGen/AMDGPU/fdiv.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdiv.bf16.ll
@@ -7,30 +7,27 @@ define bfloat @v_fdiv_bf16(bfloat %x, bfloat %y) {
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX1250-TRUE16-NEXT: v_div_scale_f32 v0, null, v2, v2, v1
-; GFX1250-TRUE16-NEXT: v_div_scale_f32 v4, vcc_lo, v1, v2, v1
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(TRANS32_DEP_1)
-; GFX1250-TRUE16-NEXT: v_rcp_f32_e32 v3, v0
+; GFX1250-TRUE16-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: v_div_scale_f32 v2, null, v1, v1, v0
+; GFX1250-TRUE16-NEXT: v_div_scale_f32 v4, vcc_lo, v0, v1, v0
+; GFX1250-TRUE16-NEXT: v_rcp_f32_e32 v3, v2
; GFX1250-TRUE16-NEXT: s_denorm_mode 15
; GFX1250-TRUE16-NEXT: v_nop
-; GFX1250-TRUE16-NEXT: v_fma_f32 v5, -v0, v3, 1.0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_fma_f32 v5, -v2, v3, 1.0
; GFX1250-TRUE16-NEXT: v_fmac_f32_e32 v3, v5, v3
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_mul_f32_e32 v5, v4, v3
+; GFX1250-TRUE16-NEXT: v_fma_f32 v6, -v2, v5, v4
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_fma_f32 v6, -v0, v5, v4
; GFX1250-TRUE16-NEXT: v_fmac_f32_e32 v5, v6, v3
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_fma_f32 v0, -v0, v5, v4
+; GFX1250-TRUE16-NEXT: v_fma_f32 v2, -v2, v5, v4
; GFX1250-TRUE16-NEXT: s_denorm_mode 12
-; GFX1250-TRUE16-NEXT: v_div_fmas_f32 v0, v0, v3, v5
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_div_fixup_f32 v0, v0, v2, v1
+; GFX1250-TRUE16-NEXT: v_div_fmas_f32 v2, v2, v3, v5
+; GFX1250-TRUE16-NEXT: v_div_fixup_f32 v0, v2, v1, v0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
index 91b0f2e55aade..f01879d98da41 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
@@ -11949,7 +11949,7 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -11963,22 +11963,20 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -12097,7 +12095,7 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -12112,21 +12110,19 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -12403,34 +12399,34 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-TRUE16-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -12555,11 +12551,12 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -12570,21 +12567,19 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -12869,34 +12864,34 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-TRUE16-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -13022,11 +13017,12 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -13037,21 +13033,19 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -13333,45 +13327,45 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-TRUE16-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -13482,46 +13476,45 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -13785,45 +13778,45 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-TRUE16-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -13935,46 +13928,45 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -14239,28 +14231,27 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -14359,28 +14350,27 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -14610,33 +14600,32 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14725,34 +14714,33 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no
; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -14971,7 +14959,7 @@ define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -14985,20 +14973,18 @@ define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
@@ -15114,7 +15100,7 @@ define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -15129,19 +15115,17 @@ define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
@@ -15411,34 +15395,34 @@ define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -15565,11 +15549,12 @@ define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -15580,21 +15565,19 @@ define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -15879,46 +15862,46 @@ define void @flat_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16030,46 +16013,45 @@ define void @flat_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-LABEL: flat_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
index ddc889f8075d5..6737e2ce3c72c 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
@@ -9807,7 +9807,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -9821,22 +9821,20 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -9955,7 +9953,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -9970,21 +9968,19 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -10262,34 +10258,34 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -10414,11 +10410,12 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -10429,21 +10426,19 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -10729,34 +10724,34 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -10882,11 +10877,12 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -10897,21 +10893,19 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -11194,7 +11188,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -11208,20 +11202,18 @@ define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
@@ -11337,7 +11329,7 @@ define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -11352,19 +11344,17 @@ define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
@@ -11632,45 +11622,45 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -11781,46 +11771,45 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -12085,45 +12074,45 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-TRUE16-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -12235,46 +12224,45 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -12540,28 +12528,27 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -12660,28 +12647,27 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -12912,33 +12898,32 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -13027,34 +13012,33 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -13277,34 +13261,34 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -13431,11 +13415,12 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -13446,21 +13431,19 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -13746,46 +13729,46 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-TRUE16-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -13897,46 +13880,45 @@ define void @flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-LABEL: flat_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
index 1b3fd173ab7b5..61dd7bedfaee1 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
@@ -9807,7 +9807,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -9821,22 +9821,20 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -9955,7 +9953,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -9970,21 +9968,19 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -10262,34 +10258,34 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -10414,11 +10410,12 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -10429,21 +10426,19 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -10729,34 +10724,34 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -10882,11 +10877,12 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -10897,21 +10893,19 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -11194,7 +11188,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -11208,20 +11202,18 @@ define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
@@ -11337,7 +11329,7 @@ define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -11352,19 +11344,17 @@ define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
@@ -11632,45 +11622,45 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -11781,46 +11771,45 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -12085,45 +12074,45 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-TRUE16-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -12235,46 +12224,45 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -12540,28 +12528,27 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -12660,28 +12647,27 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -12912,33 +12898,32 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -13027,34 +13012,33 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -13277,34 +13261,34 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -13431,11 +13415,12 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -13446,21 +13431,19 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -13746,46 +13729,46 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-TRUE16-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -13897,46 +13880,45 @@ define void @flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-LABEL: flat_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
index 53612827cd2a3..1924751427da0 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
@@ -9380,7 +9380,7 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -9394,22 +9394,20 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -9528,7 +9526,7 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -9543,21 +9541,19 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -9834,34 +9830,34 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -9986,11 +9982,12 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -10001,21 +9998,19 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -10300,34 +10295,34 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -10453,11 +10448,12 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -10468,21 +10464,19 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -10764,7 +10758,7 @@ define void @flat_agent_atomic_fsub_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -10778,20 +10772,18 @@ define void @flat_agent_atomic_fsub_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
@@ -10907,7 +10899,7 @@ define void @flat_agent_atomic_fsub_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_noret_bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -10922,19 +10914,17 @@ define void @flat_agent_atomic_fsub_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
@@ -11201,45 +11191,45 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -11350,46 +11340,45 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_noret_bf16__offset12b_pos:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -11653,45 +11642,45 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -11803,46 +11792,45 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_noret_bf16__offset12b_neg:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -12107,28 +12095,27 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -12227,28 +12214,27 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -12478,33 +12464,32 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -12593,34 +12578,33 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr %ptr,
; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_noret_bf16__offset12b__align4_pos:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -12842,34 +12826,34 @@ define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -12996,11 +12980,12 @@ define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -13011,21 +12996,19 @@ define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -13310,46 +13293,46 @@ define void @flat_system_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-TRUE16-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -13461,46 +13444,45 @@ define void @flat_system_atomic_fsub_noret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX11-TRUE16-LABEL: flat_system_atomic_fsub_noret_bf16__offset12b_pos:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
index 448a3bcb038c8..9b968d0b4dc3d 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
@@ -2521,13 +2521,13 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi(ptr inreg %sbase,
; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, 0, 0x5040100
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16hi_zero_hi:
-; GFX1250-GISEL-FAKE16: ; %bb.0:
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3]
-; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_zero_hi:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_zero_hi:
; GFX1250-SDAG-TRUE16: ; %bb.0:
@@ -2539,14 +2539,6 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi(ptr inreg %sbase,
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16hi_zero_hi:
-; GFX1250-GISEL-TRUE16: ; %bb.0:
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3]
-; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: v_lshl_or_b32 v0, v0, 16, 0
-; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX1250-NOECC-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_zero_hi:
; GFX1250-NOECC-SDAG-TRUE16: ; %bb.0:
; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -2581,13 +2573,13 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi_immneg128(ptr inr
; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, 0, 0x5040100
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16hi_zero_hi_immneg128:
-; GFX1250-GISEL-FAKE16: ; %bb.0:
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
-; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_zero_hi_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_zero_hi_immneg128:
; GFX1250-SDAG-TRUE16: ; %bb.0:
@@ -2599,14 +2591,6 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi_immneg128(ptr inr
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16hi_zero_hi_immneg128:
-; GFX1250-GISEL-TRUE16: ; %bb.0:
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
-; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: v_lshl_or_b32 v0, v0, 16, 0
-; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX1250-NOECC-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_zero_hi_immneg128:
; GFX1250-NOECC-SDAG-TRUE16: ; %bb.0:
; GFX1250-NOECC-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
diff --git a/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll b/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll
index 9a3dc507f295b..fa10543bc64c8 100644
--- a/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll
@@ -1909,41 +1909,57 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX11-SDAG-TRUE16-LABEL: v_max3_bf16_maximumnum_maximumnum__v_v_v_0:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v0.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.h, v1.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v1.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v4
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v1.h, s0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v0.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v3.h, s0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.h
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v1
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.h, v3.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, v3.h, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: v_max3_bf16_maximumnum_maximumnum__v_v_v_0:
@@ -1996,47 +2012,64 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v0.h, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.h, v1.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v3
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v4
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v1.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.h
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3
; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v1.h, s0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v0.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v3.h, s0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.h
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v1
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.h, v3.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v0
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, v3.h, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: v_max3_bf16_maximumnum_maximumnum__v_v_v_0:
@@ -2436,69 +2469,102 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.h, v1.h, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v0.h, v1.h, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.h, v1.h, v3.h, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.h
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v4
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v3.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.l, s1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.h, v0.l, v1.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.h, v3.h, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.l, v5.h, s2
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v5.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v5.h
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3
-; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.h, v5.h, s0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.l, v2.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.h, v3.h, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.h
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v0
-; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.h, s0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.l, v2.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v8.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v7
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v6, v8
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v3.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s2, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v4.l, v3.l, s1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, v3.h, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.h
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v1
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.h, v3.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v1
-; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.h, v3.h, s0
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v2.h, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v2.h, v1.l, s1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s2
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v6
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v7
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s2, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v1.l, s1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0:
@@ -2586,82 +2652,111 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.h, v1.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v0.h, v1.h, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.h, v1.h, v3.h, s0
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.h
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v4
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v3.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.l, s1
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.h, v0.l, v1.l, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.h, v3.h, s0
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.l, v5.h, s2
-; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v3
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v5.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v5.h
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3
-; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.h, v5.h, s0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v8.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v7
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v6, v8
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.l, v2.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v3.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.h, v3.h, s0
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.h
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v0
-; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s2, s0
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.h, s0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v4.l, v3.l, s1
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.l, v2.l, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v2.h, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, v3.h, s0
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.h
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v2.h, v1.l, s1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s2
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v6
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v7
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.h, v3.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v1
-; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s2, s0
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.h, v3.h, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v1.l, s1
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0:
diff --git a/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll b/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll
index e3522d452414a..c3f5baf292c99 100644
--- a/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll
@@ -69,16 +69,21 @@ define bfloat @v_test_fmed3_r_i_i_bf16_minimumnum_maximumnum(bfloat %a) #1 {
; GFX11-SDAG-TRUE16-LABEL: v_test_fmed3_r_i_i_bf16_minimumnum_maximumnum:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v1
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, 0x4000, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, 2.0, v1
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, 0x4000, v1.h, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 4.0, v1
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4080, v1.h, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4080, v0.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
%max = call bfloat @llvm.maximumnum.bf16(bfloat %a, bfloat 2.0)
%med = call bfloat @llvm.minimumnum.bf16(bfloat %max, bfloat 4.0)
@@ -191,26 +196,35 @@ define <2 x bfloat> @v_test_fmed3_r_i_i_v2bf16_minimumnum_maximumnum(<2 x bfloat
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v1
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.h, 0x4000, v0.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f32_e64 s0, v1, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, 2.0, v2
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0x4000, v2.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.h, 0x4000, v0.l, s0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f32_e64 s0, v2, v2
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0x4000, v0.h, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, 2.0, v2
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v2.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, 2.0, v3
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0x4000, v1.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 4.0, v2
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 4.0, v3
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, 0x4080, v1.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 4.0, v2
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4080, v0.l, s0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
%max = call <2 x bfloat> @llvm.maximumnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> splat (bfloat 2.0))
diff --git a/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll b/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll
index 4f6369078c386..e3f3bc7de0d85 100644
--- a/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll
@@ -1911,41 +1911,57 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX11-SDAG-TRUE16-LABEL: v_min3_bf16_minimumnum_minimumnum__v_v_v_0:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v0.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.h, v1.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v1.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v4
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v1.h, s0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v0.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v3.h, s0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.h
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v1
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.h, v3.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, v3.h, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: v_min3_bf16_minimumnum_minimumnum__v_v_v_0:
@@ -1998,47 +2014,64 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v0.h, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.h, v1.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v3
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v4
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v1.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.h
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3
; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v1.h, s0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v0.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v3.h, s0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.h
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v1
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.h, v3.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v0
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, v3.h, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: v_min3_bf16_minimumnum_minimumnum__v_v_v_0:
@@ -2441,69 +2474,102 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.h, v1.h, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v0.h, v1.h, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.h, v1.h, v3.h, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.h
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v4
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v3.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.l, s1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.h, v0.l, v1.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.h, v3.h, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.l, v5.h, s2
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v5.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.h
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3
-; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.h, v5.h, s0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.l, v2.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.h, v3.h, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.h
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v0
-; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.h, s0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.l, v2.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v8.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v7
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v6, v8
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v3.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s2, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v4.l, v3.l, s1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, v3.h, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.h
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v1
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.h, v3.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v1
-; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.h, v3.h, s0
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v2.h, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v2.h, v1.l, s1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s2
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v6
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v7
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s2, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v1.l, s1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0:
@@ -2591,82 +2657,111 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.h, v1.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v0.h, v1.h, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.h, v1.h, v3.h, s0
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.h
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v4
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v3.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.l, s1
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v5.h, v0.l, v1.l, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.h, v3.h, s0
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.l, v5.h, s2
-; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v3
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v5.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.h
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3
-; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.h, v5.h, s0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v8.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v7
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v6, v8
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v0.l, v2.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v3.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.h, v3.h, s0
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.h
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v0
-; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s2, s0
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v3.h, s0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v4.l, v3.l, s1
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.l, v2.l, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v2.h, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, v3.h, s0
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.h
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v2.h, v1.l, s1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s2
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v6
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v7
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.h, v3.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v1
-; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s2, s0
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.h, v3.h, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v1.l, s1
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0:
diff --git a/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll b/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
index 90d782e22bba0..597547ac8345e 100644
--- a/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
@@ -2671,17 +2671,17 @@ define <4 x i8> @test_signed_v4f16_v4i8(<4 x half> %f) {
; GFX11-TRUE16-NEXT: v_med3_i16 v1.h, v1.h, v2.l, 0x7f
; GFX11-TRUE16-NEXT: v_med3_i16 v0.l, v0.l, v2.l, 0x7f
; GFX11-TRUE16-NEXT: v_med3_i16 v0.h, v0.h, v2.l, 0x7f
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v2
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_signed_v4f16_v4i8:
@@ -2732,17 +2732,17 @@ define <4 x i8> @test_signed_v4f16_v4i8(<4 x half> %f) {
; GFX12-TRUE16-NEXT: v_med3_i16 v1.h, v1.h, v2.l, 0x7f
; GFX12-TRUE16-NEXT: v_med3_i16 v0.l, v0.l, v2.l, 0x7f
; GFX12-TRUE16-NEXT: v_med3_i16 v0.h, v0.h, v2.l, 0x7f
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
; GFX12-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
; GFX12-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v0.h
; GFX12-TRUE16-NEXT: v_or_b16 v2.h, v1.l, v1.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v0.h
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.h
-; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v2
+; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_signed_v4f16_v4i8:
@@ -4325,45 +4325,45 @@ define <8 x i8> @test_signed_v8f16_v8i8(<8 x half> %f) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0xff80
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, v1.l
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v1.h
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.l, v3.l
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.h, v3.h
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, v1.l
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v1.h
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, v2.l
+; GFX11-TRUE16-NEXT: v_med3_i16 v3.l, v3.l, v4.l, 0x7f
+; GFX11-TRUE16-NEXT: v_med3_i16 v3.h, v3.h, v4.l, 0x7f
; GFX11-TRUE16-NEXT: v_med3_i16 v1.l, v1.l, v4.l, 0x7f
; GFX11-TRUE16-NEXT: v_med3_i16 v1.h, v1.h, v4.l, 0x7f
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.h, v2.h
-; GFX11-TRUE16-NEXT: v_med3_i16 v3.l, v3.l, v4.l, 0x7f
-; GFX11-TRUE16-NEXT: v_med3_i16 v3.h, v3.h, v4.l, 0x7f
-; GFX11-TRUE16-NEXT: v_med3_i16 v2.l, v2.l, v4.l, 0x7f
+; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v3.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
+; GFX11-TRUE16-NEXT: v_med3_i16 v2.l, v2.l, v4.l, 0x7f
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
+; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v3.l, v3.h
+; GFX11-TRUE16-NEXT: v_med3_i16 v3.l, v2.h, v4.l, 0x7f
+; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v1.l, v1.h
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
-; GFX11-TRUE16-NEXT: v_med3_i16 v2.h, v2.h, v4.l, 0x7f
-; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v3.h
-; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_med3_i16 v0.l, v0.l, v4.l, 0x7f
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
; GFX11-TRUE16-NEXT: v_med3_i16 v0.h, v0.h, v4.l, 0x7f
-; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, 0
-; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v3.l, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v8.h
-; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v2.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v9.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v8.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v9.h
+; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v1.l, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[5:6]
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_signed_v8f16_v8i8:
@@ -4428,45 +4428,45 @@ define <8 x i8> @test_signed_v8f16_v8i8(<8 x half> %f) {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0xff80
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, v1.l
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v1.h
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.l, v3.l
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.h, v3.h
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, v1.l
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v1.h
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, v2.l
+; GFX12-TRUE16-NEXT: v_med3_i16 v3.l, v3.l, v4.l, 0x7f
+; GFX12-TRUE16-NEXT: v_med3_i16 v3.h, v3.h, v4.l, 0x7f
; GFX12-TRUE16-NEXT: v_med3_i16 v1.l, v1.l, v4.l, 0x7f
; GFX12-TRUE16-NEXT: v_med3_i16 v1.h, v1.h, v4.l, 0x7f
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.h, v2.h
-; GFX12-TRUE16-NEXT: v_med3_i16 v3.l, v3.l, v4.l, 0x7f
-; GFX12-TRUE16-NEXT: v_med3_i16 v3.h, v3.h, v4.l, 0x7f
-; GFX12-TRUE16-NEXT: v_med3_i16 v2.l, v2.l, v4.l, 0x7f
+; GFX12-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v3.h
; GFX12-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
+; GFX12-TRUE16-NEXT: v_med3_i16 v2.l, v2.l, v4.l, 0x7f
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
+; GFX12-TRUE16-NEXT: v_or_b16 v6.h, v3.l, v3.h
+; GFX12-TRUE16-NEXT: v_med3_i16 v3.l, v2.h, v4.l, 0x7f
+; GFX12-TRUE16-NEXT: v_or_b16 v2.h, v1.l, v1.h
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v0.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
-; GFX12-TRUE16-NEXT: v_med3_i16 v2.h, v2.h, v4.l, 0x7f
-; GFX12-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v3.h
-; GFX12-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX12-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX12-TRUE16-NEXT: v_med3_i16 v0.l, v0.l, v4.l, 0x7f
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
; GFX12-TRUE16-NEXT: v_med3_i16 v0.h, v0.h, v4.l, 0x7f
-; GFX12-TRUE16-NEXT: v_or_b16 v8.h, v1.l, v1.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v2.h
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, 0
-; GFX12-TRUE16-NEXT: v_or_b16 v9.h, v3.l, v3.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.h
; GFX12-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v0.h
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v8.h
-; GFX12-TRUE16-NEXT: v_or_b16 v9.l, v2.l, v1.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v8.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v9.h
-; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.l
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v8.h
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v9.h
+; GFX12-TRUE16-NEXT: v_or_b16 v6.l, v1.l, v1.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[5:6]
+; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v7
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_signed_v8f16_v8i8:
@@ -6467,23 +6467,23 @@ define <4 x i8> @test_signed_v4f32_v4i8(<4 x float> %f) {
; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v3, v3
; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v2
; GFX11-TRUE16-NEXT: s_movk_i32 s0, 0xff80
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v4, v0
; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX11-TRUE16-NEXT: v_med3_i32 v3, v3, s0, 0x7f
-; GFX11-TRUE16-NEXT: v_med3_i32 v2, v2, s0, 0x7f
+; GFX11-TRUE16-NEXT: v_med3_i32 v4, v2, s0, 0x7f
; GFX11-TRUE16-NEXT: v_med3_i32 v1, v1, s0, 0x7f
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v3.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_med3_i32 v2, v4, s0, 0x7f
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v2
+; GFX11-TRUE16-NEXT: v_med3_i32 v0, v0, s0, 0x7f
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v3.l
+; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v4.l
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
+; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v2.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_signed_v4f32_v4i8:
@@ -6526,24 +6526,24 @@ define <4 x i8> @test_signed_v4f32_v4i8(<4 x float> %f) {
; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v3, v3
; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v2
; GFX12-TRUE16-NEXT: s_movk_i32 s0, 0xff80
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v4, v0
; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_med3_i32 v3, v3, s0, 0x7f
-; GFX12-TRUE16-NEXT: v_med3_i32 v2, v2, s0, 0x7f
+; GFX12-TRUE16-NEXT: v_med3_i32 v4, v2, s0, 0x7f
; GFX12-TRUE16-NEXT: v_med3_i32 v1, v1, s0, 0x7f
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v3.l
-; GFX12-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
-; GFX12-TRUE16-NEXT: v_med3_i32 v2, v4, s0, 0x7f
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX12-TRUE16-NEXT: v_or_b16 v2.h, v0.h, v0.l
-; GFX12-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.h
-; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v2
+; GFX12-TRUE16-NEXT: v_med3_i32 v0, v0, s0, 0x7f
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v3.l
+; GFX12-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v4.l
+; GFX12-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
+; GFX12-TRUE16-NEXT: v_or_b16 v2.h, v2.h, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
+; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_signed_v4f32_v4i8:
diff --git a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
index 758cec4632bd6..af1a8872f2995 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
@@ -2461,19 +2461,19 @@ define <4 x i8> @test_unsigned_v4f16_v4i8(<4 x half> %f) {
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v1.l
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v0.h
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
; GFX11-TRUE16-NEXT: v_min_u16 v1.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_min_u16 v1.l, 0xff, v1.l
; GFX11-TRUE16-NEXT: v_min_u16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_min_u16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v2
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_unsigned_v4f16_v4i8:
@@ -2515,19 +2515,19 @@ define <4 x i8> @test_unsigned_v4f16_v4i8(<4 x half> %f) {
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v1.l
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v0.h
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
; GFX12-TRUE16-NEXT: v_min_u16 v1.h, 0xff, v1.h
; GFX12-TRUE16-NEXT: v_min_u16 v1.l, 0xff, v1.l
; GFX12-TRUE16-NEXT: v_min_u16 v0.h, 0xff, v0.h
; GFX12-TRUE16-NEXT: v_min_u16 v0.l, 0xff, v0.l
; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v0.h
; GFX12-TRUE16-NEXT: v_or_b16 v2.h, v1.l, v1.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v0.h
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.h
-; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v2
+; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_unsigned_v4f16_v4i8:
@@ -4024,38 +4024,38 @@ define <8 x i8> @test_unsigned_v8f16_v8i8(<8 x half> %f) {
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v1.h
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.h, v3.h
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v1.l
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.h, v2.h
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, v3.l
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.h, v2.h
; GFX11-TRUE16-NEXT: v_min_u16 v1.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_min_u16 v3.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, v2.l
; GFX11-TRUE16-NEXT: v_min_u16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
-; GFX11-TRUE16-NEXT: v_min_u16 v2.h, 0xff, v2.h
; GFX11-TRUE16-NEXT: v_min_u16 v3.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v3.h
-; GFX11-TRUE16-NEXT: v_min_u16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT: v_min_u16 v4.l, 0xff, v2.h
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v0.h
+; GFX11-TRUE16-NEXT: v_min_u16 v4.h, 0xff, v2.l
+; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v1.l, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v3.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v4.l
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, v0.l
; GFX11-TRUE16-NEXT: v_min_u16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, 0
-; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v3.l, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.h
+; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v4.h, v1.l
; GFX11-TRUE16-NEXT: v_min_u16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v8.h
-; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v2.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v9.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v8.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v9.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v8
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[7:8]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v8.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_unsigned_v8f16_v8i8:
@@ -4116,38 +4116,38 @@ define <8 x i8> @test_unsigned_v8f16_v8i8(<8 x half> %f) {
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v1.h
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.h, v3.h
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v1.l
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.h, v2.h
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, v3.l
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.h, v2.h
; GFX12-TRUE16-NEXT: v_min_u16 v1.h, 0xff, v1.h
; GFX12-TRUE16-NEXT: v_min_u16 v3.h, 0xff, v3.h
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, v2.l
; GFX12-TRUE16-NEXT: v_min_u16 v1.l, 0xff, v1.l
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v0.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
-; GFX12-TRUE16-NEXT: v_min_u16 v2.h, 0xff, v2.h
; GFX12-TRUE16-NEXT: v_min_u16 v3.l, 0xff, v3.l
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
; GFX12-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v3.h
-; GFX12-TRUE16-NEXT: v_min_u16 v2.l, 0xff, v2.l
+; GFX12-TRUE16-NEXT: v_min_u16 v4.l, 0xff, v2.h
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v0.h
+; GFX12-TRUE16-NEXT: v_min_u16 v4.h, 0xff, v2.l
+; GFX12-TRUE16-NEXT: v_or_b16 v2.h, v1.l, v1.h
+; GFX12-TRUE16-NEXT: v_or_b16 v8.h, v3.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v4.l
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, v0.l
; GFX12-TRUE16-NEXT: v_min_u16 v0.h, 0xff, v0.h
-; GFX12-TRUE16-NEXT: v_or_b16 v8.h, v1.l, v1.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v2.h
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, 0
-; GFX12-TRUE16-NEXT: v_or_b16 v9.h, v3.l, v3.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.h
+; GFX12-TRUE16-NEXT: v_or_b16 v8.l, v4.h, v1.l
; GFX12-TRUE16-NEXT: v_min_u16 v0.l, 0xff, v0.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v0.h
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v8.h
-; GFX12-TRUE16-NEXT: v_or_b16 v9.l, v2.l, v1.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v8.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v9.h
-; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.l
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v8.h
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v9.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v8
+; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[7:8]
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v8.h
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_unsigned_v8f16_v8i8:
@@ -6018,22 +6018,22 @@ define <4 x i8> @test_unsigned_v4f32_v4i8(<4 x float> %f) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v4, v1
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v4, v2
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX11-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v3
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v2
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v3.l
; GFX11-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v4
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v2.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v2
+; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_unsigned_v4f32_v4i8:
@@ -6070,22 +6070,22 @@ define <4 x i8> @test_unsigned_v4f32_v4i8(<4 x float> %f) {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v4, v1
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v4, v2
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX12-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v3
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v2
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v3.l
; GFX12-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v4
-; GFX12-TRUE16-NEXT: v_or_b16 v2.h, v2.l, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.h
-; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v2
+; GFX12-TRUE16-NEXT: v_or_b16 v2.h, v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
+; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_unsigned_v4f32_v4i8:
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
index c4707cf1a56eb..8810505e87e72 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
@@ -13756,7 +13756,7 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -13770,22 +13770,20 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -13904,7 +13902,7 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -13919,21 +13917,19 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -14351,34 +14347,34 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -14503,11 +14499,12 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -14518,21 +14515,19 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -14962,34 +14957,34 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -15115,11 +15110,12 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -15130,21 +15126,19 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -15567,7 +15561,7 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -15581,20 +15575,18 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
@@ -15710,7 +15702,7 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -15725,19 +15717,17 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
@@ -16144,45 +16134,45 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16293,46 +16283,45 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -16740,45 +16729,45 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16890,46 +16879,45 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -17315,28 +17303,27 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -17435,28 +17422,27 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -17796,33 +17782,32 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17911,34 +17896,33 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -18290,34 +18274,34 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-TRUE16-NEXT: .LBB62_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -18444,11 +18428,12 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -18459,21 +18444,19 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -18901,46 +18884,46 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-TRUE16-NEXT: .LBB63_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -19052,46 +19035,45 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-LABEL: global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB63_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
index 124f67a3787c9..5b41f57ca9563 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
@@ -8698,7 +8698,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -8712,22 +8712,20 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -8846,7 +8844,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -8861,21 +8859,19 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -9204,34 +9200,34 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -9356,11 +9352,12 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -9371,21 +9368,19 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -9724,34 +9719,34 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -9877,11 +9872,12 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -9892,21 +9888,19 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -10242,7 +10236,7 @@ define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -10256,20 +10250,18 @@ define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
@@ -10385,7 +10377,7 @@ define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -10400,19 +10392,17 @@ define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
@@ -10731,45 +10721,45 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -10880,46 +10870,45 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -11236,45 +11225,45 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-TRUE16-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -11386,46 +11375,45 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -11743,28 +11731,27 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -11863,28 +11850,27 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -12157,33 +12143,32 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -12272,34 +12257,33 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -12561,34 +12545,34 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -12715,11 +12699,12 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -12730,21 +12715,19 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -13083,46 +13066,46 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-TRUE16-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -13234,46 +13217,45 @@ define void @global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-LABEL: global_system_atomic_fmax_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
index d524e952c5da2..7b529d1aff4f4 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
@@ -8698,7 +8698,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -8712,22 +8712,20 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -8846,7 +8844,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -8861,21 +8859,19 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -9204,34 +9200,34 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -9356,11 +9352,12 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -9371,21 +9368,19 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -9724,34 +9719,34 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -9877,11 +9872,12 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -9892,21 +9888,19 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -10242,7 +10236,7 @@ define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -10256,20 +10250,18 @@ define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
@@ -10385,7 +10377,7 @@ define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -10400,19 +10392,17 @@ define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
@@ -10731,45 +10721,45 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -10880,46 +10870,45 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -11236,45 +11225,45 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-TRUE16-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -11386,46 +11375,45 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -11743,28 +11731,27 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -11863,28 +11850,27 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -12157,33 +12143,32 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -12272,34 +12257,33 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -12561,34 +12545,34 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -12715,11 +12699,12 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -12730,21 +12715,19 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -13083,46 +13066,46 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-TRUE16-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -13234,46 +13217,45 @@ define void @global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-LABEL: global_system_atomic_fmin_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
index ef422fb1d61fe..2031374ca0df4 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
@@ -9220,7 +9220,7 @@ define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -9234,22 +9234,20 @@ define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -9368,7 +9366,7 @@ define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -9383,21 +9381,19 @@ define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -9724,34 +9720,34 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -9876,11 +9872,12 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -9891,21 +9888,19 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -10242,34 +10237,34 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -10395,11 +10390,12 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -10410,21 +10406,19 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -10758,7 +10752,7 @@ define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -10772,20 +10766,18 @@ define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
@@ -10901,7 +10893,7 @@ define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_noret_bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
@@ -10916,19 +10908,17 @@ define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
@@ -11245,45 +11235,45 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -11394,46 +11384,45 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1)
; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_pos:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -11748,45 +11737,45 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -11898,46 +11887,45 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b_neg(ptr addrspace(1)
; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b_neg:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -12253,28 +12241,27 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr addr
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -12373,28 +12360,27 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr addr
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -12665,33 +12651,32 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr addr
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -12780,34 +12765,33 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr addr
; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -13067,34 +13051,34 @@ define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -13221,11 +13205,12 @@ define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -13236,21 +13221,19 @@ define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -13587,46 +13570,46 @@ define void @global_system_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-TRUE16-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -13738,46 +13721,45 @@ define void @global_system_atomic_fsub_noret_bf16__offset12b_pos(ptr addrspace(1
; GFX11-TRUE16-LABEL: global_system_atomic_fsub_noret_bf16__offset12b_pos:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v7
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
diff --git a/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll b/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
index a707de96eb116..a090a1b804677 100644
--- a/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
@@ -59,9 +59,10 @@ define void @test_i8load_v4i8store(ptr addrspace(1) %ptr_a, ptr addrspace(1) %pt
; GCN-SDAG-REAL16-NEXT: v_lshlrev_b16 v0.l, 8, v6.l
; GCN-SDAG-REAL16-NEXT: s_wait_loadcnt 0x0
; GCN-SDAG-REAL16-NEXT: v_perm_b32 v1, v10, v7, 0xc0c0004
-; GCN-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GCN-SDAG-REAL16-NEXT: v_or_b16 v0.h, v6.l, v0.l
-; GCN-SDAG-REAL16-NEXT: v_mov_b16_e32 v0.l, 0
+; GCN-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GCN-SDAG-REAL16-NEXT: v_or_b16 v0.l, v6.l, v0.l
+; GCN-SDAG-REAL16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GCN-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GCN-SDAG-REAL16-NEXT: v_or_b32_e32 v0, v1, v0
; GCN-SDAG-REAL16-NEXT: global_store_b32 v[8:9], v0, off
; GCN-SDAG-REAL16-NEXT: s_set_pc_i64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll b/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll
index 11fdce249ba78..ca986a91b9162 100644
--- a/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll
@@ -4379,9 +4379,9 @@ define amdgpu_ps <2 x half> @global_load_saddr_i16_d16hi_zero_hi(ptr addrspace(1
;
; GFX12-GISEL-TRUE16-LABEL: global_load_saddr_i16_d16hi_zero_hi:
; GFX12-GISEL-TRUE16: ; %bb.0:
-; GFX12-GISEL-TRUE16-NEXT: global_load_d16_hi_b16 v0, v0, s[2:3]
+; GFX12-GISEL-TRUE16-NEXT: global_load_d16_b16 v0, v0, s[2:3]
; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0
+; GFX12-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
;
; GFX12-GISEL-FAKE16-LABEL: global_load_saddr_i16_d16hi_zero_hi:
@@ -4441,9 +4441,9 @@ define amdgpu_ps <2 x half> @global_load_saddr_i16_d16hi_zero_hi_immneg128(ptr a
;
; GFX12-GISEL-TRUE16-LABEL: global_load_saddr_i16_d16hi_zero_hi_immneg128:
; GFX12-GISEL-TRUE16: ; %bb.0:
-; GFX12-GISEL-TRUE16-NEXT: global_load_d16_hi_b16 v0, v0, s[2:3] offset:-128
+; GFX12-GISEL-TRUE16-NEXT: global_load_d16_b16 v0, v0, s[2:3] offset:-128
; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0
+; GFX12-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
;
; GFX12-GISEL-FAKE16-LABEL: global_load_saddr_i16_d16hi_zero_hi_immneg128:
diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll
index 8345a2802d924..7298f72c19ebf 100644
--- a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll
+++ b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll
@@ -853,91 +853,47 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_cc_half(half inre
}
define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_cc_bfloat(bfloat inreg %a, bfloat %b) {
- ; DAGISEL-GFX11-WF32-TRUE16-LABEL: name: amdgpu_cs_chain_cc_bfloat
- ; DAGISEL-GFX11-WF32-TRUE16: bb.0 (%ir-block.0):
- ; DAGISEL-GFX11-WF32-TRUE16-NEXT: liveins: $sgpr0, $vgpr8
- ; DAGISEL-GFX11-WF32-TRUE16-NEXT: {{ $}}
- ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_MOV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_MOV_B16_t16_e64 0, 0, 0, implicit $exec
- ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE killed [[V_MOV_B16_t16_e64_]], %subreg.lo16, [[COPY]], %subreg.hi16
- ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc
- ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[S_LSHL_B32_]], 0, killed [[REG_SEQUENCE]], 0, 0, implicit $mode, implicit $exec
- ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec
- ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767
- ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 killed [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], killed [[S_MOV_B32_]], implicit $exec
- ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304
- ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_ADD_F32_e64_]], killed [[S_MOV_B32_1]], implicit $exec
- ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec
- ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD3_U32_e64_]], 0, killed [[V_OR_B32_e64_]], killed [[V_CMP_U_F32_e64_]], implicit $exec
- ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
- ; DAGISEL-GFX11-WF32-TRUE16-NEXT: FLAT_STORE_SHORT_D16_HI killed [[COPY2]], killed [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
- ; DAGISEL-GFX11-WF32-TRUE16-NEXT: S_ENDPGM 0
- ;
- ; DAGISEL-GFX11-WF32-FAKE16-LABEL: name: amdgpu_cs_chain_cc_bfloat
- ; DAGISEL-GFX11-WF32-FAKE16: bb.0 (%ir-block.0):
- ; DAGISEL-GFX11-WF32-FAKE16-NEXT: liveins: $sgpr0, $vgpr8
- ; DAGISEL-GFX11-WF32-FAKE16-NEXT: {{ $}}
- ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec
- ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc
- ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[S_LSHL_B32_]], 0, killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec
- ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec
- ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767
- ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 killed [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], killed [[S_MOV_B32_]], implicit $exec
- ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304
- ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_ADD_F32_e64_]], killed [[S_MOV_B32_1]], implicit $exec
- ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec
- ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD3_U32_e64_]], 0, killed [[V_OR_B32_e64_]], killed [[V_CMP_U_F32_e64_]], implicit $exec
- ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
- ; DAGISEL-GFX11-WF32-FAKE16-NEXT: FLAT_STORE_SHORT_D16_HI killed [[COPY2]], killed [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
- ; DAGISEL-GFX11-WF32-FAKE16-NEXT: S_ENDPGM 0
- ;
- ; DAGISEL-GFX11-WF64-TRUE16-LABEL: name: amdgpu_cs_chain_cc_bfloat
- ; DAGISEL-GFX11-WF64-TRUE16: bb.0 (%ir-block.0):
- ; DAGISEL-GFX11-WF64-TRUE16-NEXT: liveins: $sgpr0, $vgpr8
- ; DAGISEL-GFX11-WF64-TRUE16-NEXT: {{ $}}
- ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_MOV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_MOV_B16_t16_e64 0, 0, 0, implicit $exec
- ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE killed [[V_MOV_B16_t16_e64_]], %subreg.lo16, [[COPY]], %subreg.hi16
- ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc
- ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[S_LSHL_B32_]], 0, killed [[REG_SEQUENCE]], 0, 0, implicit $mode, implicit $exec
- ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec
- ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767
- ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 killed [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], killed [[S_MOV_B32_]], implicit $exec
- ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304
- ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_ADD_F32_e64_]], killed [[S_MOV_B32_1]], implicit $exec
- ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_64_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec
- ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD3_U32_e64_]], 0, killed [[V_OR_B32_e64_]], killed [[V_CMP_U_F32_e64_]], implicit $exec
- ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
- ; DAGISEL-GFX11-WF64-TRUE16-NEXT: FLAT_STORE_SHORT_D16_HI killed [[COPY2]], killed [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
- ; DAGISEL-GFX11-WF64-TRUE16-NEXT: S_ENDPGM 0
+ ; DAGISEL-GFX11-WF32-LABEL: name: amdgpu_cs_chain_cc_bfloat
+ ; DAGISEL-GFX11-WF32: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX11-WF32-NEXT: liveins: $sgpr0, $vgpr8
+ ; DAGISEL-GFX11-WF32-NEXT: {{ $}}
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[S_LSHL_B32_]], 0, killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 killed [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], killed [[S_MOV_B32_]], implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_ADD_F32_e64_]], killed [[S_MOV_B32_1]], implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD3_U32_e64_]], 0, killed [[V_OR_B32_e64_]], killed [[V_CMP_U_F32_e64_]], implicit $exec
+ ; DAGISEL-GFX11-WF32-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; DAGISEL-GFX11-WF32-NEXT: FLAT_STORE_SHORT_D16_HI killed [[COPY2]], killed [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
+ ; DAGISEL-GFX11-WF32-NEXT: S_ENDPGM 0
;
- ; DAGISEL-GFX11-WF64-FAKE16-LABEL: name: amdgpu_cs_chain_cc_bfloat
- ; DAGISEL-GFX11-WF64-FAKE16: bb.0 (%ir-block.0):
- ; DAGISEL-GFX11-WF64-FAKE16-NEXT: liveins: $sgpr0, $vgpr8
- ; DAGISEL-GFX11-WF64-FAKE16-NEXT: {{ $}}
- ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec
- ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc
- ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[S_LSHL_B32_]], 0, killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec
- ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec
- ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767
- ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 killed [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], killed [[S_MOV_B32_]], implicit $exec
- ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304
- ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_ADD_F32_e64_]], killed [[S_MOV_B32_1]], implicit $exec
- ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_64_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec
- ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD3_U32_e64_]], 0, killed [[V_OR_B32_e64_]], killed [[V_CMP_U_F32_e64_]], implicit $exec
- ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
- ; DAGISEL-GFX11-WF64-FAKE16-NEXT: FLAT_STORE_SHORT_D16_HI killed [[COPY2]], killed [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
- ; DAGISEL-GFX11-WF64-FAKE16-NEXT: S_ENDPGM 0
+ ; DAGISEL-GFX11-WF64-LABEL: name: amdgpu_cs_chain_cc_bfloat
+ ; DAGISEL-GFX11-WF64: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX11-WF64-NEXT: liveins: $sgpr0, $vgpr8
+ ; DAGISEL-GFX11-WF64-NEXT: {{ $}}
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, killed [[S_LSHL_B32_]], 0, killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 killed [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], killed [[S_MOV_B32_]], implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_ADD_F32_e64_]], killed [[S_MOV_B32_1]], implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_64_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, killed [[V_ADD3_U32_e64_]], 0, killed [[V_OR_B32_e64_]], killed [[V_CMP_U_F32_e64_]], implicit $exec
+ ; DAGISEL-GFX11-WF64-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; DAGISEL-GFX11-WF64-NEXT: FLAT_STORE_SHORT_D16_HI killed [[COPY2]], killed [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
+ ; DAGISEL-GFX11-WF64-NEXT: S_ENDPGM 0
;
; DAGISEL-GFX10-WF32-LABEL: name: amdgpu_cs_chain_cc_bfloat
; DAGISEL-GFX10-WF32: bb.0 (%ir-block.0):
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll
index c1f3a12dba578..9bcf8c7db29a0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ballot.i32.ll
@@ -593,21 +593,12 @@ exit:
}
define amdgpu_cs i32 @compare_bfloats(bfloat %x, bfloat %y) {
-; GFX10-LABEL: compare_bfloats:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: v_cmp_gt_f32_e64 s0, v0, v1
-; GFX10-NEXT: ; return to shader part epilog
-;
-; GFX11-LABEL: compare_bfloats:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-NEXT: v_cmp_gt_f32_e64 s0, v1, v2
-; GFX11-NEXT: ; return to shader part epilog
+; CHECK-LABEL: compare_bfloats:
+; CHECK: ; %bb.0:
+; CHECK-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; CHECK-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; CHECK-NEXT: v_cmp_gt_f32_e64 s0, v0, v1
+; CHECK-NEXT: ; return to shader part epilog
%cmp = fcmp ogt bfloat %x, %y
%ballot = call i32 @llvm.amdgcn.ballot.i32(i1 %cmp)
ret i32 %ballot
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll
index 06b404e9f2c05..cb513eb8cdeeb 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll
@@ -16,15 +16,14 @@ define bfloat @v_exp2_bf16(bfloat %in) {
; GFX1200-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v1
+; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 0x42800000, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v1, v0
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0
; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
@@ -91,25 +90,26 @@ define bfloat @v_exp2_fabs_bf16(bfloat %in) {
; GFX1200-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1200-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.l
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v1
+; GFX1200-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0x7fff, v0.l
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 0x42800000, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v1, v0
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -169,25 +169,26 @@ define bfloat @v_exp2_fneg_fabs_bf16(bfloat %in) {
; GFX1200-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1200-SDAG-TRUE16-NEXT: v_or_b16 v1.h, 0x8000, v0.l
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v1
+; GFX1200-SDAG-TRUE16-NEXT: v_or_b16 v0.l, 0x8000, v0.l
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 0x42800000, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v1, v0
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -248,25 +249,26 @@ define bfloat @v_exp2_fneg_bf16(bfloat %in) {
; GFX1200-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1200-SDAG-TRUE16-NEXT: v_xor_b16 v1.h, 0x8000, v0.l
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v1
+; GFX1200-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, 0x8000, v0.l
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 0x42800000, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v1, v0
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -326,15 +328,14 @@ define bfloat @v_exp2_bf16_fast(bfloat %in) {
; GFX1200-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v1
+; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 0x42800000, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v1, v0
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0
; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
@@ -515,43 +516,43 @@ define <2 x bfloat> @v_exp2_fabs_v2bf16(<2 x bfloat> %in) {
; GFX1200-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1200-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.l
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0x7fff, v0.l
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX1200-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0x7fff, v0.h
+; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v1
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v2, v1, v2
-; GFX1200-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.h
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0xc2fc0000, v1
-; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 0x42800000, s0
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0xffffffc0, s0
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v1, v0
-; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v1, v2
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0xffffffc0, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
+; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v1, v1, v3
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0xffffffc0, vcc_lo
; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_2) | instid1(VALU_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v1, v1, v2
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v1, v1
; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v3
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v1, v1, v2
+; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -636,43 +637,43 @@ define <2 x bfloat> @v_exp2_fneg_fabs_v2bf16(<2 x bfloat> %in) {
; GFX1200-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1200-SDAG-TRUE16-NEXT: v_or_b16 v1.h, 0x8000, v0.l
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_or_b16 v0.l, 0x8000, v0.l
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX1200-SDAG-TRUE16-NEXT: v_or_b16 v0.l, 0x8000, v0.h
+; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v1
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v2, v1, v2
-; GFX1200-SDAG-TRUE16-NEXT: v_or_b16 v1.h, 0x8000, v0.h
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0xc2fc0000, v1
-; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 0x42800000, s0
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0xffffffc0, s0
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v1, v0
-; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v1, v2
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0xffffffc0, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
+; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v1, v1, v3
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0xffffffc0, vcc_lo
; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_2) | instid1(VALU_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v1, v1, v2
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v1, v1
; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v3
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v1, v1, v2
+; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -758,43 +759,43 @@ define <2 x bfloat> @v_exp2_fneg_v2bf16(<2 x bfloat> %in) {
; GFX1200-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1200-SDAG-TRUE16-NEXT: v_xor_b16 v1.h, 0x8000, v0.l
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, 0x8000, v0.l
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX1200-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, 0x8000, v0.h
+; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v1
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v2, v1, v2
-; GFX1200-SDAG-TRUE16-NEXT: v_xor_b16 v1.h, 0x8000, v0.h
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0xc2fc0000, v1
-; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 0x42800000, s0
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0xffffffc0, s0
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v1, v0
-; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v1, v2
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0xffffffc0, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
+; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v1, v1, v3
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0xffffffc0, vcc_lo
; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_2) | instid1(VALU_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v1, v1, v2
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v1, v1
; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v3
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v1, v1, v2
+; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log.ll b/llvm/test/CodeGen/AMDGPU/llvm.log.ll
index 3fbd6b104e615..af6665c79feef 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log.ll
@@ -6174,99 +6174,28 @@ define float @v_log_f32_from_fpext_bf16(bfloat %src) {
; GFX900-NEXT: v_sub_f32_e32 v0, v0, v1
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1100-SDAG-TRUE16-LABEL: v_log_f32_from_fpext_bf16:
-; GFX1100-SDAG-TRUE16: ; %bb.0:
-; GFX1100-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1100-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v1
-; GFX1100-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 32, vcc_lo
-; GFX1100-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v1, v0
-; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-TRUE16-NEXT: v_log_f32_e32 v0, v0
-; GFX1100-SDAG-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-SDAG-TRUE16-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0
-; GFX1100-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1100-SDAG-TRUE16-NEXT: v_fma_f32 v2, 0x3f317217, v0, -v1
-; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-TRUE16-NEXT: v_fmamk_f32 v2, v0, 0x3377d1cf, v2
-; GFX1100-SDAG-TRUE16-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0
-; GFX1100-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo
-; GFX1100-SDAG-TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1100-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1100-SDAG-FAKE16-LABEL: v_log_f32_from_fpext_bf16:
-; GFX1100-SDAG-FAKE16: ; %bb.0:
-; GFX1100-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-SDAG-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo
-; GFX1100-SDAG-FAKE16-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-FAKE16-NEXT: v_log_f32_e32 v0, v0
-; GFX1100-SDAG-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-SDAG-FAKE16-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0
-; GFX1100-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1100-SDAG-FAKE16-NEXT: v_fma_f32 v2, 0x3f317217, v0, -v1
-; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-FAKE16-NEXT: v_fmamk_f32 v2, v0, 0x3377d1cf, v2
-; GFX1100-SDAG-FAKE16-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0
-; GFX1100-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo
-; GFX1100-SDAG-FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1100-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1100-GISEL-TRUE16-LABEL: v_log_f32_from_fpext_bf16:
-; GFX1100-GISEL-TRUE16: ; %bb.0:
-; GFX1100-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1100-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1100-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v1
-; GFX1100-GISEL-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 32, vcc_lo
-; GFX1100-GISEL-TRUE16-NEXT: v_ldexp_f32 v0, v1, v0
-; GFX1100-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1100-GISEL-TRUE16-NEXT: v_log_f32_e32 v0, v0
-; GFX1100-GISEL-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-TRUE16-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0
-; GFX1100-GISEL-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1100-GISEL-TRUE16-NEXT: v_fma_f32 v2, 0x3f317217, v0, -v1
-; GFX1100-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-TRUE16-NEXT: v_fmamk_f32 v2, v0, 0x3377d1cf, v2
-; GFX1100-GISEL-TRUE16-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1100-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-TRUE16-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0
-; GFX1100-GISEL-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo
-; GFX1100-GISEL-TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1100-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1100-GISEL-FAKE16-LABEL: v_log_f32_from_fpext_bf16:
-; GFX1100-GISEL-FAKE16: ; %bb.0:
-; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo
-; GFX1100-GISEL-FAKE16-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0
-; GFX1100-GISEL-FAKE16-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_f32 v2, 0x3f317217, v0, -v1
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-FAKE16-NEXT: v_fmamk_f32 v2, v0, 0x3377d1cf, v2
-; GFX1100-GISEL-FAKE16-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0
-; GFX1100-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo
-; GFX1100-GISEL-FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX1100-LABEL: v_log_f32_from_fpext_bf16:
+; GFX1100: ; %bb.0:
+; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1100-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
+; GFX1100-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo
+; GFX1100-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1100-NEXT: v_log_f32_e32 v0, v0
+; GFX1100-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX1100-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0
+; GFX1100-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
+; GFX1100-NEXT: v_fma_f32 v2, 0x3f317217, v0, -v1
+; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-NEXT: v_fmamk_f32 v2, v0, 0x3377d1cf, v2
+; GFX1100-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1100-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0
+; GFX1100-NEXT: v_cndmask_b32_e64 v1, 0, 0x41b17218, vcc_lo
+; GFX1100-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX1100-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log_f32_from_fpext_bf16:
; R600: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log10.ll b/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
index 60b34391430e7..bbf19ff267b01 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
@@ -6174,99 +6174,28 @@ define float @v_log10_f32_from_fpext_bf16(bfloat %src) {
; GFX900-NEXT: v_sub_f32_e32 v0, v0, v1
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1100-SDAG-TRUE16-LABEL: v_log10_f32_from_fpext_bf16:
-; GFX1100-SDAG-TRUE16: ; %bb.0:
-; GFX1100-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1100-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v1
-; GFX1100-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 32, vcc_lo
-; GFX1100-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v1, v0
-; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-TRUE16-NEXT: v_log_f32_e32 v0, v0
-; GFX1100-SDAG-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-SDAG-TRUE16-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0
-; GFX1100-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1100-SDAG-TRUE16-NEXT: v_fma_f32 v2, 0x3e9a209a, v0, -v1
-; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-TRUE16-NEXT: v_fmamk_f32 v2, v0, 0x3284fbcf, v2
-; GFX1100-SDAG-TRUE16-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0
-; GFX1100-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo
-; GFX1100-SDAG-TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1100-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1100-SDAG-FAKE16-LABEL: v_log10_f32_from_fpext_bf16:
-; GFX1100-SDAG-FAKE16: ; %bb.0:
-; GFX1100-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-SDAG-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo
-; GFX1100-SDAG-FAKE16-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-FAKE16-NEXT: v_log_f32_e32 v0, v0
-; GFX1100-SDAG-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-SDAG-FAKE16-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0
-; GFX1100-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1100-SDAG-FAKE16-NEXT: v_fma_f32 v2, 0x3e9a209a, v0, -v1
-; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-FAKE16-NEXT: v_fmamk_f32 v2, v0, 0x3284fbcf, v2
-; GFX1100-SDAG-FAKE16-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0
-; GFX1100-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo
-; GFX1100-SDAG-FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1100-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1100-GISEL-TRUE16-LABEL: v_log10_f32_from_fpext_bf16:
-; GFX1100-GISEL-TRUE16: ; %bb.0:
-; GFX1100-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1100-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1100-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v1
-; GFX1100-GISEL-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 32, vcc_lo
-; GFX1100-GISEL-TRUE16-NEXT: v_ldexp_f32 v0, v1, v0
-; GFX1100-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1100-GISEL-TRUE16-NEXT: v_log_f32_e32 v0, v0
-; GFX1100-GISEL-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-TRUE16-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0
-; GFX1100-GISEL-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1100-GISEL-TRUE16-NEXT: v_fma_f32 v2, 0x3e9a209a, v0, -v1
-; GFX1100-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-TRUE16-NEXT: v_fmamk_f32 v2, v0, 0x3284fbcf, v2
-; GFX1100-GISEL-TRUE16-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1100-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-TRUE16-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0
-; GFX1100-GISEL-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo
-; GFX1100-GISEL-TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1100-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1100-GISEL-FAKE16-LABEL: v_log10_f32_from_fpext_bf16:
-; GFX1100-GISEL-FAKE16: ; %bb.0:
-; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo
-; GFX1100-GISEL-FAKE16-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0
-; GFX1100-GISEL-FAKE16-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
-; GFX1100-GISEL-FAKE16-NEXT: v_fma_f32 v2, 0x3e9a209a, v0, -v1
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-FAKE16-NEXT: v_fmamk_f32 v2, v0, 0x3284fbcf, v2
-; GFX1100-GISEL-FAKE16-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0
-; GFX1100-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo
-; GFX1100-GISEL-FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX1100-LABEL: v_log10_f32_from_fpext_bf16:
+; GFX1100: ; %bb.0:
+; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1100-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
+; GFX1100-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo
+; GFX1100-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1100-NEXT: v_log_f32_e32 v0, v0
+; GFX1100-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX1100-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0
+; GFX1100-NEXT: v_cmp_gt_f32_e64 s0, 0x7f800000, |v0|
+; GFX1100-NEXT: v_fma_f32 v2, 0x3e9a209a, v0, -v1
+; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1100-NEXT: v_fmamk_f32 v2, v0, 0x3284fbcf, v2
+; GFX1100-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1100-NEXT: v_cndmask_b32_e64 v0, v0, v1, s0
+; GFX1100-NEXT: v_cndmask_b32_e64 v1, 0, 0x411a209b, vcc_lo
+; GFX1100-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX1100-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log10_f32_from_fpext_bf16:
; R600: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
index 7035d143ff679..bc81ab88a1a13 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
@@ -3656,67 +3656,20 @@ define float @v_log2_f32_from_fpext_bf16(bfloat %src) {
; GFX900-NEXT: v_sub_f32_e32 v0, v0, v1
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1100-SDAG-TRUE16-LABEL: v_log2_f32_from_fpext_bf16:
-; GFX1100-SDAG-TRUE16: ; %bb.0:
-; GFX1100-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1100-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v1
-; GFX1100-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 32, vcc_lo
-; GFX1100-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v1, v0
-; GFX1100-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
-; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1100-SDAG-TRUE16-NEXT: v_log_f32_e32 v0, v0
-; GFX1100-SDAG-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-SDAG-TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1100-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1100-SDAG-FAKE16-LABEL: v_log2_f32_from_fpext_bf16:
-; GFX1100-SDAG-FAKE16: ; %bb.0:
-; GFX1100-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-SDAG-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 32, vcc_lo
-; GFX1100-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
-; GFX1100-SDAG-FAKE16-NEXT: v_ldexp_f32 v0, v0, v2
-; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1100-SDAG-FAKE16-NEXT: v_log_f32_e32 v0, v0
-; GFX1100-SDAG-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-SDAG-FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1100-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1100-GISEL-TRUE16-LABEL: v_log2_f32_from_fpext_bf16:
-; GFX1100-GISEL-TRUE16: ; %bb.0:
-; GFX1100-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1100-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1100-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1100-GISEL-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v1
-; GFX1100-GISEL-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 32, vcc_lo
-; GFX1100-GISEL-TRUE16-NEXT: v_ldexp_f32 v0, v1, v0
-; GFX1100-GISEL-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
-; GFX1100-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1100-GISEL-TRUE16-NEXT: v_log_f32_e32 v0, v0
-; GFX1100-GISEL-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1100-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1100-GISEL-FAKE16-LABEL: v_log2_f32_from_fpext_bf16:
-; GFX1100-GISEL-FAKE16: ; %bb.0:
-; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1100-GISEL-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 32, vcc_lo
-; GFX1100-GISEL-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
-; GFX1100-GISEL-FAKE16-NEXT: v_ldexp_f32 v0, v0, v2
-; GFX1100-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1100-GISEL-FAKE16-NEXT: v_log_f32_e32 v0, v0
-; GFX1100-GISEL-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-GISEL-FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1100-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX1100-LABEL: v_log2_f32_from_fpext_bf16:
+; GFX1100: ; %bb.0:
+; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1100-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
+; GFX1100-NEXT: v_cndmask_b32_e64 v2, 0, 32, vcc_lo
+; GFX1100-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
+; GFX1100-NEXT: v_ldexp_f32 v0, v0, v2
+; GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-NEXT: v_log_f32_e32 v0, v0
+; GFX1100-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX1100-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX1100-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_log2_f32_from_fpext_bf16:
; R600: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-bf16.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-bf16.ll
index 6826ab50e7466..a1e78bdc3b27f 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-bf16.ll
@@ -565,29 +565,15 @@ define float @v_mad_mix_f32_preextractfabsfneg_bf16hi_bf16lo_bf16lo(i32 %src0.ar
}
define float @v_mad_mix_f32_bf16lo_bf16lo_bf16lo_all_cast_from_half(half %src0, half %src1, half %src2) #0 {
-; GFX1250-FAKE16-LABEL: v_mad_mix_f32_bf16lo_bf16lo_bf16lo_all_cast_from_half:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_dual_lshlrev_b32 v3, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_fmac_f32_e32 v0, v3, v1
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-REAL16-LABEL: v_mad_mix_f32_bf16lo_bf16lo_bf16lo_all_cast_from_half:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v4.h, v1.l
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX1250-REAL16-NEXT: v_fmac_f32_e32 v0, v3, v4
-; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_mad_mix_f32_bf16lo_bf16lo_bf16lo_all_cast_from_half:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v3, 16, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_fmac_f32_e32 v0, v3, v1
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%src0.bf16 = bitcast half %src0 to bfloat
%src1.bf16 = bitcast half %src1 to bfloat
%src2.bf16 = bitcast half %src2 to bfloat
@@ -599,24 +585,14 @@ define float @v_mad_mix_f32_bf16lo_bf16lo_bf16lo_all_cast_from_half(half %src0,
}
define float @v_mad_mix_f32_bf16lo_cast_from_half_bf16lo_bf16lo(half %src0, bfloat %src1, bfloat %src2) #0 {
-; GFX1250-FAKE16-LABEL: v_mad_mix_f32_bf16lo_cast_from_half_bf16lo_bf16lo:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_fma_mix_f32_bf16 v0, v0, v1, v2 op_sel_hi:[0,1,1]
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-REAL16-LABEL: v_mad_mix_f32_bf16lo_cast_from_half_bf16lo_bf16lo:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v3.l, 0
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-REAL16-NEXT: v_fma_mix_f32_bf16 v0, v3, v1, v2 op_sel_hi:[0,1,1]
-; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_mad_mix_f32_bf16lo_cast_from_half_bf16lo_bf16lo:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, v0, v1, v2 op_sel_hi:[0,1,1]
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%src0.bf16 = bitcast half %src0 to bfloat
%src0.ext = fpext bfloat %src0.bf16 to float
%src1.ext = fpext bfloat %src1 to float
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll
index 5884b35a31ec4..67a85c6d53b24 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll
@@ -73,22 +73,14 @@ define <2 x bfloat> @v_mad_mixhi_bf16_bf16lo_bf16lo_bf16lo_reglo(bfloat %src0, b
}
define i32 @v_mad_mixhi_bf16_bf16lo_bf16lo_bf16lo_intpack(bfloat %src0, bfloat %src1, bfloat %src2) #0 {
-; GFX1250-FAKE16-LABEL: v_mad_mixhi_bf16_bf16lo_bf16lo_bf16lo_intpack:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_fma_mixlo_bf16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-REAL16-LABEL: v_mad_mixhi_bf16_bf16lo_bf16lo_bf16lo_intpack:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_fma_mixhi_bf16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.l, 0
-; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_mad_mixhi_bf16_bf16lo_bf16lo_bf16lo_intpack:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_fma_mixlo_bf16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%src0.ext = fpext bfloat %src0 to float
%src1.ext = fpext bfloat %src1 to float
%src2.ext = fpext bfloat %src2 to float
@@ -101,22 +93,14 @@ define i32 @v_mad_mixhi_bf16_bf16lo_bf16lo_bf16lo_intpack(bfloat %src0, bfloat %
}
define i32 @v_mad_mixhi_bf16_bf16lo_bf16lo_bf16lo_intpack_sext(bfloat %src0, bfloat %src1, bfloat %src2) #0 {
-; GFX1250-FAKE16-LABEL: v_mad_mixhi_bf16_bf16lo_bf16lo_bf16lo_intpack_sext:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_fma_mixlo_bf16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-REAL16-LABEL: v_mad_mixhi_bf16_bf16lo_bf16lo_bf16lo_intpack_sext:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_fma_mixhi_bf16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.l, 0
-; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_mad_mixhi_bf16_bf16lo_bf16lo_bf16lo_intpack_sext:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_fma_mixlo_bf16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%src0.ext = fpext bfloat %src0 to float
%src1.ext = fpext bfloat %src1 to float
%src2.ext = fpext bfloat %src2 to float
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
index 611bbfce607c2..8fc6eefac2b84 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
@@ -220,20 +220,13 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo(half %src0, half %src
}
define i32 @v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack(half %src0, half %src1, half %src2) #0 {
-; SDAG-GFX11-TRUE16-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack:
-; SDAG-GFX11-TRUE16: ; %bb.0:
-; SDAG-GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX11-TRUE16-NEXT: v_fma_mixhi_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
-; SDAG-GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0
-; SDAG-GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; SDAG-GFX11-FAKE16-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack:
-; SDAG-GFX11-FAKE16: ; %bb.0:
-; SDAG-GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX11-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
-; SDAG-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SDAG-GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX11-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_fma_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SDAG-GFX11-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-GFX9-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack:
; SDAG-GFX9: ; %bb.0:
@@ -302,20 +295,13 @@ define i32 @v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack(half %src0, half %src1, ha
}
define i32 @v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext(half %src0, half %src1, half %src2) #0 {
-; SDAG-GFX11-TRUE16-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext:
-; SDAG-GFX11-TRUE16: ; %bb.0:
-; SDAG-GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX11-TRUE16-NEXT: v_fma_mixhi_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
-; SDAG-GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0
-; SDAG-GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; SDAG-GFX11-FAKE16-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext:
-; SDAG-GFX11-FAKE16: ; %bb.0:
-; SDAG-GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX11-FAKE16-NEXT: v_fma_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
-; SDAG-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SDAG-GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX11-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_fma_mixlo_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SDAG-GFX11-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-GFX9-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_intpack_sext:
; SDAG-GFX9: ; %bb.0:
@@ -666,4 +652,3 @@ attributes #0 = { nounwind denormal_fpenv(float: preservesign) }
attributes #1 = { nounwind readnone speculatable }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GISEL-GFX11-FAKE16: {{.*}}
-; SDAG-GFX11: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
index c7acbb0584904..b382e53ddbae2 100644
--- a/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
@@ -107,25 +107,31 @@ define bfloat @v_maximumnum_bf16(bfloat %x, bfloat %y) {
; GFX11-TRUE16-LABEL: v_maximumnum_bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v0.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v1.h, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.h, v1.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v1.h, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_bf16:
@@ -159,28 +165,34 @@ define bfloat @v_maximumnum_bf16(bfloat %x, bfloat %y) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v0.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.h
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v1.h, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_bf16:
@@ -288,18 +300,19 @@ define bfloat @v_maximumnum_bf16_nnan(bfloat %x, bfloat %y) {
; GFX11-TRUE16-LABEL: v_maximumnum_bf16_nnan:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v0.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v2
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_bf16_nnan:
@@ -325,20 +338,20 @@ define bfloat @v_maximumnum_bf16_nnan(bfloat %x, bfloat %y) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.l, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v2
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_bf16_nnan:
@@ -543,38 +556,52 @@ define <2 x bfloat> @v_maximumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v1.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.h, v2.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v2.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.h, v2.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.l, v4.h, s2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.h, v4.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v4.h, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v2.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v2.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v2bf16:
@@ -628,44 +655,56 @@ define <2 x bfloat> @v_maximumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.h, v2.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v2.h
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v2.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.h, v2.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.l, v4.h, s2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.h
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v6
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.h, v4.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v2.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v4.h, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v2.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v2bf16:
@@ -847,24 +886,29 @@ define <2 x bfloat> @v_maximumnum_v2bf16_nnan(<2 x bfloat> %x, <2 x bfloat> %y)
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v4, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v0.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.h, v0.h, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.h, v0.h, s0
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.h, v0.h, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v4
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v0.h, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v2bf16_nnan:
@@ -905,27 +949,31 @@ define <2 x bfloat> @v_maximumnum_v2bf16_nnan(<2 x bfloat> %x, <2 x bfloat> %y)
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v4, v3
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v0.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.h, v0.h, s1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.h, v0.h, s0
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v4
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.h, v0.h, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v0.h, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v2bf16_nnan:
@@ -1214,54 +1262,67 @@ define <3 x bfloat> @v_maximumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v8, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.h, v2.h, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v2.h, v4.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v4.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v5.h, v4.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.l, v1.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v1.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.h, v1.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.l, v0.h, s2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v4.h, v0.h, s0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v4.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v9
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v10, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v4.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s3, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s4, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v3bf16:
@@ -1330,62 +1391,72 @@ define <3 x bfloat> @v_maximumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v8, v8
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.h, v2.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v2.h, v4.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.h
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v4.l, s1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v8
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v9
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v10, v11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v4.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v4.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s3, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v5.h, v4.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.l, v1.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.h, v1.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.l, v0.h, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s4, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v4.h, v0.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v3bf16:
@@ -1631,33 +1702,40 @@ define <3 x bfloat> @v_maximumnum_v3bf16_nnan(<3 x bfloat> %x, <3 x bfloat> %y)
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v6
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v9, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v2.h, v0.h, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v5, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.h, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.h, v0.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.h, v0.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.h, v0.h, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v7
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s3, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s4, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, s1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v3bf16_nnan:
@@ -1706,37 +1784,43 @@ define <3 x bfloat> @v_maximumnum_v3bf16_nnan(<3 x bfloat> %x, <3 x bfloat> %y)
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v0
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v6
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v9, v8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v2.h, v0.h, s1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v5, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.h, v1.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v5, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v7
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s3, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.h, v0.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.h, v0.h, s1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s4, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.h, v0.h, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, s1
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v3bf16_nnan:
@@ -2115,70 +2199,77 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v1.h, v3.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.h, v6.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v6.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v6.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v6.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v0.h, v2.h, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v2.h, v6.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v6.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v10, v10
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v1.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.h, v6.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v3.l, v7.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v7.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v7, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v7.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v6.h, v7.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.l, v0.h, s2
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.h, v0.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v1 :: v_dual_mov_b32 v1, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v4.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v4.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v1.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v10, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v9
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v12, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v4.l, s5
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v10
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v6.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v4bf16:
@@ -2271,81 +2362,86 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v1.h, v3.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.h, v6.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v6.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v6.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v6.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v0.h, v2.h, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v2.h, v6.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v5
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v6.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v10, v10
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v4.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v4.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v6.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v9
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v10, v8
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v9
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v12, v13
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v1.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.h, v6.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v3.l, v7.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v7.h
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v7.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v4.l, s5
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v10
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v6.h, v7.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.l, v0.h, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v6.l, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.h, v0.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v1 :: v_dual_mov_b32 v1, v4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v4bf16:
@@ -2669,42 +2765,45 @@ define <4 x bfloat> @v_maximumnum_v4bf16_nnan(<4 x bfloat> %x, <4 x bfloat> %y)
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v6
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v9, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v11, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.h, v1.h, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v6, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v0.h, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v1.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v7
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v9
; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.h, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.h, v1.h, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v5, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.h, v1.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.h, v0.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.h, v0.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.h, v0.h, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s5, s6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v0.h, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v1.h, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v4bf16_nnan:
@@ -2769,47 +2868,49 @@ define <4 x bfloat> @v_maximumnum_v4bf16_nnan(<4 x bfloat> %x, <4 x bfloat> %y)
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v0
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v6
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v9, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v11, v10
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.h, v1.h, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v6, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.h
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.h, v1.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.h, v1.h, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v5, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.h, v1.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v5, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v0.h, s2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v1.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v7
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v8
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v9
; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.h
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.h, v0.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.h, v0.h, s1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s5, s6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.h, v0.h, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v0.h, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v1.h, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v4bf16_nnan:
@@ -3360,101 +3461,112 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, 0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v8.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v2.h, v5.h, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v5.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v11, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v5.h, v8.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v8.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v8.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v8.h, s0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v7, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v1.h, v4.h, s1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v4.h, v8.h, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.h, v8.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v8.h
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v5.h, v6.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v12, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v1.h, v4.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v13, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v14, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v8.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v15, v15
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v10, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v0.h, v3.h, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v4.h, v8.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v16, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v3.h, v9.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v9.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v17, v17
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v11.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v5.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v12, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v16
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v8.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s6
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v15, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v7.l, v6.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v4.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v11.l, v9.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v10.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v14, v13
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v16, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v12
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v7
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.h, v8.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v0.h, v3.h, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v3.h, v8.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v8.h
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v9.h, v8.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v2.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v9.h, v8.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v5.l, v10.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v10.h
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v10, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v10.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v8.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v8.h, v10.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v4.l, v1.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v1.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v8.h, v1.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v3.l, v0.h, s2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v7
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v8.h, v0.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v8.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s2, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s0, s1
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v11
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v2.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v7
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v12
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v1.l
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v10.l, v8.l, s5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v9.l, s6
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, s3
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v2.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v6bf16:
@@ -3584,118 +3696,124 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, 0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v8.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v5
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v2.h, v5.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v5.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v11, v11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v5.h, v8.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v8.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v8.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v8.h, s0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v7, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v5.h, v6.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v12, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v8
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v1.h, v4.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v13, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v14, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v3
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v8.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v15, v15
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v10, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v0.h, v3.h, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v1.h, v4.h, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v4.h, v8.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v7
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.h, v8.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v8.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v7
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.h, v8.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v0.h, v3.h, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v3.h, v8.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v8.h
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v9
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v9.h, v8.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v2.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v9.h, v8.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v5.l, v10.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v10.h
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v10, v8
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v10.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v8.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v4.h, v8.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v3.h, v9.l, s2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v10.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v9.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v17, v17
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v7.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v11.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v5.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v12, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v16
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v14
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v8.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s6
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v15, v12
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v8.h, v10.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v4.l, v1.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v7.l, v6.l, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v4.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v11.l, v9.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v0.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v10.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v7
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v14, v13
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v16, v15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v12
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v7
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v8.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v9.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-TRUE16-NEXT: s_and_b32 s5, s2, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s6, s0, s1
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v11
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v2.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v7
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v12
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v1.l
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v8.h, v1.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v3.l, v0.h, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v7
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v10.l, v8.l, s5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v9.l, s6
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, s3
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v8.h, v0.h, s0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v2.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v6bf16:
@@ -4472,132 +4590,154 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v14, v14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v3.h, v7.h, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v11.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v7.h, v11.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v11.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v11.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v11.h, s0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v2.h, v6.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v6.h, v11.h, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v11.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v9.h, v11.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v11.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v9
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v9.h, v11.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v1.h, v5.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v5.h, v11.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v12, v12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v11.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v10.h, v11.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v11.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v3.h, v7.h, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.h, v6.h, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v7.h, v8.l, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v9.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v6.h, v9.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v10.h, v11.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v0.h, v4.h, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v4.h, v11.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v11.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v10.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v1.h, v5.h, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v11.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v5.h, v12.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v8.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v9.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v17, v14
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v15, v16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v13.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v8.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v9.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v10.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v11.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v15, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v19
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v12.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v0.h, v4.h, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v15
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v13.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v4.h, v14.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v14.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v7.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v15.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v18
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s1, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v17
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v12.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v10.l, v8.l, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s4, s3
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v16, v17
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v11.l, v9.l, s3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v13.l, v12.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v15.l, v14.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v15, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v6.l, v9.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v12.h, v11.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v11.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v3.l, v7.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v12.h, v11.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v7.l, v13.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v6
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v13.h
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v13, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v12, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v11.h, v13.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.l, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v11.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v11.h, v13.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v6.l, v2.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v2.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v11.h, v2.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v11.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v11.h, v2.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v5.l, v1.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v11
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v11.h, v1.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v11.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v11.h, v1.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v4.l, v0.h, s2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v11.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v11
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v11.h, v0.h, s0
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v3 :: v_dual_mov_b32 v3, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v9.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v7.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v13, v12
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v16, v15
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.l, v9.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v6.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v7.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v4.l
+; GFX11-TRUE16-NEXT: s_and_b32 s7, s2, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v12
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v9.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v1.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v11
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v0.l
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, s6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v9.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v1.l, s3
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s4, s5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v14.l, s7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.l, v0.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v6.l, v3.l, s0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v2 :: v_dual_mov_b32 v2, v7
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v8bf16:
@@ -4756,154 +4896,175 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v7
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v7
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v1
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v14, v14
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v3.h, v7.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v11.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v7.h, v11.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v11.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v11.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v11.h, s0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v6
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v2.h, v6.h, s1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v6.h, v11.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v11.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v9
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v9.h, v11.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v11.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v9
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v9.h, v11.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v1.h, v5.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v5.h, v11.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v12, v12
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v11.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v10
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v10.h, v11.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v11.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v10.h, v11.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v0.h, v4.h, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v4.h, v11.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v11.h
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v12
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v3.h, v7.h, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.h, v6.h, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v7.h, v8.l, s1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v9.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v12.h, v11.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v6.h, v9.l, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v11.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v12
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v8.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v10.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v1.h, v5.h, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v11.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v5.h, v12.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v8.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v9.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v17, v14
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v15, v16
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v12.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v13.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v8.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v9.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v10.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v11.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v15, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v18
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v19
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v12.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v14
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v0.h, v4.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v15
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v13.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v4.h, v14.l, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v14.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v7.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v15.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v18
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s1, s2
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v17
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v12.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v19
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v3.l, v7.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v3.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v12.h, v11.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v7.l, v13.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v6
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v13.h
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v13, v11
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v12, v12
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v11.h, v13.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v11
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.l, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v11.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v10.l, v8.l, s2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s4, s3
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v16, v17
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v11.h, v13.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v6.l, v2.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v2.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v11.l, v9.l, s3
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v13.l, v12.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v11.h, v2.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v15.l, v14.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v11.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v11.h, v2.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v5.l, v1.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v11
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v15, v15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v11.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v9
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v6.l, v9.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v11.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v12
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v9.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v4.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v0.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v7.l, v3.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v13, v12
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v16, v15
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v10
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.l, v9.l, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v6.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v14.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v7.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v4.l
+; GFX12-TRUE16-NEXT: s_and_b32 s7, s2, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v12
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v9.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v11
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v0.l
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s3, s6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v11.h, v1.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v4.l, v0.h, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v10
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v11
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v11.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v11
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v9.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v1.l, s3
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s4, s5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v14.l, s7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v11.h, v0.h, s0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v3 :: v_dual_mov_b32 v3, v8
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.l, v0.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v6.l, v3.l, s0
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v2 :: v_dual_mov_b32 v2, v7
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v8bf16:
@@ -6316,258 +6477,310 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
; GFX11-TRUE16-LABEL: v_maximumnum_v16bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v16, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v16
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v17, v17
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v23.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v10
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, v7 :: v_dual_mov_b32 v17, v6
; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v15
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v16.h, v15.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v14
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v16
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v17
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v16.h, v15.h, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v23.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v26, v26
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v15.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v23.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v7
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.h, v23.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v6.h, v14.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.h, v14.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v18, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v23.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.h, v17.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v23.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v17
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v17.h, v14.h, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v6.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v15.h, v6.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.h, v17.h, v23.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v5.h, v13.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.h, v13.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v19, v19
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v23.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.h, v18.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v23.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v18
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.h, v18.h, v23.h, s0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v4.h, v12.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.h, v12.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v20, v20
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v23.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v19
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.h, v19.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v23.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v19
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.h, v19.h, v23.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v3.h, v11.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.h, v11.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v23.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v20
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.h, v20.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v23.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v20
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v18.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v6.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v7.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v14.h, v18.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.h, v20.h, v23.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v2.h, v10.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.h, v10.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v23.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v21
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.h, v21.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v23.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v21
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.h, v21.h, v23.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v1.h, v9.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.h, v9.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v24, v24
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v23.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v22
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.h, v22.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v23.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v22
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v18.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v19.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v20, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v5.h, v13.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v21, v23
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v13.h, v20.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v20.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v18.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v12
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v21.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v22
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v19.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v4.h, v12.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v26
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v12.h, v22.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v24, v25
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v22.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v27
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v23.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v20.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v21.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.l, v6.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v19.l, v18.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v24, v25
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v27, v27
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v26
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v23.l, v22.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v3.h, v11.h, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v10
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v18
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v6.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v11.h, v7.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v2.h, v10.h, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v7.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v24
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v18.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v10.h, v19.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v24
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v23.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v1.h, v9.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v25, v28
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v29
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v30
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v20.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v9.h, v24.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v27
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v7.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v26, v28
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v24.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v25.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v22.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v18.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v19.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v23.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v26, v27
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.h, v21.l, v20.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.h, v6.l, v22.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v29
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v25.l, v24.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v28
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v7.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v20
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v21, v21
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v8
; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.h, v22.h, v23.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v0.h, v8.h, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.h, v8.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v23.h
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v24
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.h, v24.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v14
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v24
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v16.l, v15.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v23.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v24.h, v23.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v15.l, v16.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v6
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v16.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v25, v25
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v16, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v16.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.l, v14.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v23.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v23.h, v16.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v14.l, v6.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v13
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v23
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v16, v16
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v6.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.l, v13.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v23.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v23.h, v6.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v13.l, v5.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v12
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v5.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v23
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v13, v13
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v5.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.l, v12.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v23.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v23.h, v5.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v12.l, v4.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v17
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v4.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.l, v11.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v23.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v23.h, v4.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v11.l, v3.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v18
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v3.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.l, v10.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v23.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v23.h, v3.h, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v19.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v10.l, v2.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v2.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v4, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v19
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v2.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v9.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v23.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v23.h, v2.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v9.l, v1.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v3, v3
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v20
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v1.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v21
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v8.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v23.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v23.h, v1.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v8.l, v0.h, s2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v22
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v23
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v23.h, v0.h, s0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v20
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v0.h, v8.h, s3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v16
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v18.l, v7.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v8.h, v20.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.h, v23.l, v19.l, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v20.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v15.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v17
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v24.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v16.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v14
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v6.l, v24.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v22, v23
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v16.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v17.l, v14.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v20.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v23
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v13
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v22, v21
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v6.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v13.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v23
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v16.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v17
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v15.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v20.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v22, v21
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v17
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v14.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v7.l, v20.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v22, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v4
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v15.l, v16.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v12
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v13.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v12.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v6.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v17, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s0, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v11.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v21
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v5.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v14.l, v6.l, s3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v11.l
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v13.l, v5.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v17, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v21
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v22, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v3.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v10.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v9
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v22, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v10.l, v2.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v8.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v5.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v12.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v13, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v17
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v10.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v13, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v17, v14
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v22, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v23
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v5.l, v2.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v11.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v8.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v12.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v14
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v5.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v2.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v13
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v14
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v1.l
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s6, s7
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s5, s8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v0.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v9.l, v1.l, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v12.l, v2.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v11.l, v3.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v10.l, v4.l, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v15
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v20
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v16 :: v_dual_mov_b32 v4, v19
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v18
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v16bf16:
@@ -6893,299 +7106,356 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v16, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v6
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v14
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v13
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v16
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v11
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v17, v17
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v23.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v10
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v16, v7 :: v_dual_mov_b32 v17, v6
; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v15
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v9
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v15
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v14
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v16
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v17
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v16.h, v15.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v16.h, v15.h, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v23.l
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v26, v26
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v15.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v7
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v23.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v7
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.h, v23.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v6.h, v14.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v5
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.h, v14.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v18, v18
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v23.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v17
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v17.h, v14.h, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v6.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.h, v17.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v23.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v17
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v15.h, v6.l, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.h, v17.h, v23.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v5.h, v13.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.h, v13.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v19, v19
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v23.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v18
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v18.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v6.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v7.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.h, v18.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v23.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v18
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.h, v18.h, v23.h, s0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v12
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v4.h, v12.h, s1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.h, v12.h, v23.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v14.h, v18.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v13
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v18.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v19.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v20, v20
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v23.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v19
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.h, v19.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v23.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v19
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.h, v19.h, v23.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v3.h, v11.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.h, v11.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v23.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v20
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v20, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.h, v20.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v23.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v20
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v5.h, v13.h, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.h, v20.h, v23.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v2.h, v10.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v21, v23
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.h, v10.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v23.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v21
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v13.h, v20.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v7.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v20.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v18.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v12
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v21.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v22
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v19.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.h, v21.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v23.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v21
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v4.h, v12.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v26
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v12.h, v22.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v24, v25
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v22.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v27
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v23.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v20.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX12-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s3
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v21.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.h, v21.h, v23.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v1.h, v9.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.h, v9.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v24, v24
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v23.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v22
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.l, v6.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v19.l, v18.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v24, v25
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v27, v27
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v11
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v26
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v23.l, v22.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v3.h, v11.h, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v10
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v18
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v6.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.h, v22.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v23.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v22
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.h, v22.h, v23.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v0.h, v8.h, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v16
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v11.h, v7.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v2.h, v10.h, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v7.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v24
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v18.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.h, v8.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v23.h
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v24
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v10.h, v19.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff0000, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v24
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v29.l, v19.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v30.l, v23.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.h, v24.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v14
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v24
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v1.h, v9.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v25, v28
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v29
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v30
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v20.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v16.l, v15.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v23.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v9.h, v24.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v27
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v7.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v26, v28
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v24.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v25.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v22.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v18.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v19.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s3
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v29.l, v23.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v26, v27
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v24.h, v23.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v15.l, v16.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v6
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v16.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v25, v25
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v16, v23
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v16.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.l, v14.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v23.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.h, v21.l, v20.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.h, v6.l, v22.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v29
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v25.l, v24.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v28
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v7.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v20
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v21, v21
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v6.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v8
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v19.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v20
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v0.h, v8.h, s3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v16
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v23.h, v16.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v14.l, v6.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v13
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v23
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v16, v16
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v6.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v18.l, v7.l, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v22
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.l, v13.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v23.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v8.h, v20.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v15
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v23.h, v6.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v13.l, v5.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v12
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v5.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v23
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v13, v13
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.h, v23.l, v19.l, s1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v20.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v5.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v11
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v15.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v17
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v24.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.l, v12.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v23.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v16.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v14
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v23.h, v5.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v12.l, v4.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v17
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v23
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v4.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v10
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v6.l, v24.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v22, v23
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v15.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v16.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.l, v11.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v23.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v17.l, v14.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v20.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v23
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v13
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v22, v21
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v14.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v6.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v13.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v23
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v16.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v17
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v15.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v20.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v22, v21
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v13.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v5.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v17
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v16.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v14.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v23.h, v4.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v11.l, v3.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v18
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v23
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v3.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v9
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.l, v10.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v23.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v7.l, v20.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v22, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v4
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v23.h, v3.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v10.l, v2.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v2.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v4, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v19
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v23
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v15.l, v16.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v12
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v3
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v13.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v2.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v8
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v12.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v6.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v17, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v22
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s0, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v11.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v21
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v12.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v5.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v9.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v23.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v23.h, v2.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v9.l, v1.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v3, v3
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v20
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v23
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v21
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v8.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v23.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v14.l, v6.l, s3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v11.l
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v23.h, v1.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v8.l, v0.h, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v22
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v23
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v23
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v13.l, v5.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v17, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v21
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v22, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v3.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v10.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v9
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v22, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v8
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v10.l, v2.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v8.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v5.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v12.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v13, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v17
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v9.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v8.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v0.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v10.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v13, v12
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v17, v14
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v22, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v23
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v5.l, v2.l, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v11.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v8.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v12.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v14
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v9.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v5.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v17
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v2.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v13
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v14
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v1.l
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s6, s7
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s5, s8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v23.h, v0.h, s0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v15
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v0.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v9.l, v1.l, s4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v12.l, v2.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v11.l, v3.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v10.l, v4.l, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v15
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, v20 :: v_dual_mov_b32 v3, v16
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v19 :: v_dual_mov_b32 v5, v18
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v16bf16:
@@ -10010,500 +10280,602 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
; GFX11-TRUE16-LABEL: v_maximumnum_v32bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: scratch_load_b32 v55, off, s32
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v53, v15 :: v_dual_mov_b32 v48, v13
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v31, v10
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v33, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v51, v14 :: v_dual_and_b32 v8, 0xffff0000, v53
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v34, v11 :: v_dual_mov_b32 v37, v12
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v36, v9
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v51
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v30
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v24
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v23
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v22
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v48
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v19
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v18
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v17
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v54.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v55
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v53.h, v55.h, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v54.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v55.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v8.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v51.h, v30.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v29
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v37
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v30.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v8.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v48.h, v29.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v28
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v34
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v29.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v8.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v37.h, v28.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v27
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v31
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v28.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v8.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v34.h, v27.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v26
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v36
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v27.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v8.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v31.h, v26.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v25
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v33
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v26.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v8.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v36.h, v25.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v25.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v10
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v29
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v26
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v13
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v49, v49
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v36, v36
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v50, v50
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v67, v67
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.l, v10.h, v26.h, s9
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s43, v98, v98
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff0000, v12
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v30
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v69, v69
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v67.l, v26.h, v36.l, s10
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v98.l, v36.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v37, v37
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v28
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v50.l, v4.h, v20.h, s21
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v132.l, v67.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v98
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v34, v34
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v12.h, v28.h, s5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v36.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v132
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v38, v38
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v28
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v25
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s21, v98, v132
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v65, v65
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v65.l, v28.h, v34.l, s6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v21
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v52, v52
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v67.l, v67.l, v36.l, s21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v29
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v20
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v68, v68
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v98.l, v67.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s44, v99, v99
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v70, v70
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v81, v81
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v98
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v81.l, v20.h, v50.l, s22
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v35, v35
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v23
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s21, 0, v98
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v33, v33
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v13.h, v29.h, s3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v64, v64
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s21, s5
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.h, v67.l, v36.l, s5
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v64.l, v29.h, v33.l, s4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v66, v66
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v71, v71
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.l, v2.h, v18.h, s25
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v51, v51
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v15
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v14
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v85, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.l, v9.h, v25.h, s11
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v8.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v33.h, v24.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v24.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v32, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v54.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v7.h, v23.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v23.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v35, v35
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v32
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v32.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v32
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v50.h, v32.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v6.h, v22.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v21
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v35, v35
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v22.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v32
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v32.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v32
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.h, v32.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v5.h, v21.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v20
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v35, v35
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v54.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v21.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v32
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v32.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v32
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v32.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v4.h, v20.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v14.h, v30.h, s1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v16
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v55, v55
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v68.l, v25.h, v37.l, s12
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v99.l, v37.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v37.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v55.l, v30.h, v32.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v96, v96
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v133.l, v68.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v99
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v29.l, s43
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v130.l, v65.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v133, 16, v133
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v24
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v15
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v85, v85
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v86, v86
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s22, v99, v133
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v128.l, v55.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s44
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v68.l, v68.l, v37.l, s22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v130
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v18
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v54, v54
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v87, v87
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v99.l, v68.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.l, v0.h, v16.h, s29
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v33.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v129.l, v64.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v99
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v128
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v30
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.l, v5.h, v21.h, s19
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v118.l, v13.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s22, 0, v99
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s19, v96, v130
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v29.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v82, v82
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s22, s6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v85.l, v16.h, v54.l, s40
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.h, v68.l, v37.l, s6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v27
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v87
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v97, v97
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v118
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v48, v48
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v48.l, v6.h, v22.h, s17
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s17, v86, v128
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v129
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v65.l, v65.l, v34.l, s19
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v83, v83
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v71.l, v22.h, v48.l, s18
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v102.l, v48.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0, v48.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v83.l, v18.h, v52.l, s26
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v116.l, v54.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.l, v71.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v102, 16, v102
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s18, v87, v128
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v85.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v19
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v144, 16, v144
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s43, v118, v96
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v65.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.l, v52.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s25, v102, v144
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v83.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v116
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v87
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v64.l, v64.l, v33.l, s18
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v71.l, v71.l, v48.l, s25
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v80, v80
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v80.l, v21.h, v49.l, s20
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v53, v53
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v102.l, v71.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v30.l, s41
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v114
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s41, v116, v87
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v102, 16, v102
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v55.l, v55.l, v32.l, s17
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v64.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s19, 0, v96
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.l, v8.h, v24.h, s13
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s25, 0, v102
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s29, v114, v86
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v55.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v87
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v12
+; GFX11-TRUE16-NEXT: s_and_b32 s9, s25, s9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v69.l, v24.h, v38.l, s14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v71.l, v48.l, s9
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s18, 0, v87
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s45, v100, v100
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v100.l, v38.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v39, v39
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v103.l, v49.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v112.l, v50.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s42
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v134.l, v69.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v11.h, v27.h, s7
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v145.l, v80.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v146.l, v81.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s17, 0, v86
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.l, v3.h, v19.h, s23
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v66.l, v27.h, v35.l, s8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.l, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v117.l, v14.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v100
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v103
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.l, v66.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v97
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v112
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v128.l, v30.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v134, 16, v134
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v131
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v145, 16, v145
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v146, 16, v146
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v84, 0xffff0000, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v53.l, v1.h, v17.h, s27
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s20, v97, v131
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v82.l, v19.h, v51.l, s24
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v117
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v128
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s23, v100, v134
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v66.l, v66.l, v35.l, s20
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s26, v103, v145
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s27, v112, v146
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v84, v84
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v113.l, v51.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.l, v66.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v147.l, v82.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s42, v117, v128
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v69.l, v69.l, v38.l, s23
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v80.l, v80.l, v49.l, s26
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v97
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v81.l, v81.l, v50.l, s27
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v32.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v84.l, v17.h, v53.l, s28
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v113
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s20, 0, v97
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v147, 16, v147
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s42
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v100.l, v69.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v103.l, v80.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v112.l, v81.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s46, v101, v101
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v28.l, s45
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v115.l, v53.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v129.l, v84.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s28, v113, v147
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v117.l, v30.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v100
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v103
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v112
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s17, s1
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v33.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s46
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v55.l, v32.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.l, v7.h, v23.h, s15
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v38.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0, v49.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0, v50.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v115
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v129, 16, v129
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v82.l, v82.l, v51.l, s28
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v117
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s23, 0, v100
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s26, 0, v103
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s27, 0, v112
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s18, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v130.l, v28.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v64.l, v33.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v70.l, v23.h, v39.l, s16
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0, v14.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s40, v115, v129
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v113.l, v82.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s42, 0, v117
+; GFX11-TRUE16-NEXT: s_and_b32 s7, s23, s7
+; GFX11-TRUE16-NEXT: s_and_b32 s10, s26, s10
+; GFX11-TRUE16-NEXT: s_and_b32 s11, s27, s11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.h, v69.l, v38.l, s7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.h, v80.l, v49.l, s10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.h, v81.l, v50.l, s11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v130
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v119
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v83.l, v83.l, v52.l, s29
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v84.l, v84.l, v53.l, s40
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v113
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v11
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0, v51.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.l, v83.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v115.l, v84.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s28, 0, v113
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s43
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0, v52.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v114
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v115
+; GFX11-TRUE16-NEXT: s_and_b32 s12, s28, s12
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0, v53.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v34.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s29, 0, v114
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s40, 0, v115
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v101.l, v39.l
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s19, s3
+; GFX11-TRUE16-NEXT: s_and_b32 s13, s29, s13
+; GFX11-TRUE16-NEXT: s_and_b32 s14, s40, s14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v65.l, v34.l, s3
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s20, s4
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v135.l, v70.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.h, v66.l, v35.l, s4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v101
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v85.l, v85.l, v54.l, s41
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v39.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v135, 16, v135
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0, v54.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v116.l, v85.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s24, v101, v135
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.h, v20.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v38, v38
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v54.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v35
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.h, v35.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v35
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.h, v35.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v3.h, v19.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v116
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v70.l, v70.l, v39.l, s24
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.h, v19.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v49, v49
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v54.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v38
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.h, v38.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v38
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.h, v38.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v2.h, v18.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s41, 0, v116
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v101.l, v70.l
+; GFX11-TRUE16-NEXT: s_and_b32 s15, s41, s15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v101
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.h, v85.l, v54.l, s15
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v31
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v87, 0xffff0000, v31
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v86.l, v15.h, v31.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s24, 0, v101
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v96, v96
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.l, v15.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v87, v87
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v86.l
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s24, s8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.l, v31.l, v15.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v97
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v87.l, v31.h, v86.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.h, v70.l, v39.l, s8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v99.l, v31.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v98.l, v87.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v99
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.h, v18.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v52, v52
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v54.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v49
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.h, v49.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v49
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.h, v49.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v1.h, v17.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v98
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v97, v99
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.h, v17.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v64, v64
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v54.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v52
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.h, v52.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v52
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.h, v52.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v0.h, v16.h, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v53
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v64.h, v16.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v54.h
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v64
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v64.h, v64.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v55
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v64
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v66.h, v53.l, v55.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v53.h, v64.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v55.l, v66.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v51
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v30
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v66.h
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v96, v98
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.l, v31.l, v15.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v15.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v87.l, v86.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v86.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v66, v54
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v64, v64
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v66.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v29
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.h, v51.l, v30.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v54.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v54.h, v66.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v30.l, v51.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v48
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v51.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v55, v55
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v51, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v51.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v54.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.h, v48.l, v29.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v28
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v54.h, v51.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v29.l, v30.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v37
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v30.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v48, v48
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v30, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v30.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v54.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.h, v37.l, v28.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v54.h, v30.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v28.l, v29.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v34
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v27
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v29.h
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v29, v54
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v30, v30
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v29.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v54.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.h, v34.l, v27.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v54.h, v29.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v27.l, v28.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v31
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v26
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v28.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v28, v54
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v29, v29
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v28.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v31.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v32.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v36
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v82.l, v51.l, s12
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v29.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v37
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.h, v83.l, v52.l, s13
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v38
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.h, v84.l, v53.l, s14
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v31.l, v15.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s42, s16
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v30.l, v14.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v50, v49
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v27
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v27.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v49, v49
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v26
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v51
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v11.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v26.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v49, v49
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v48
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v28.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v10.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v50, v50
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v25
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v27.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v32.l, v86.l, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v26.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v25.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v50, v50
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v13.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v52
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v53
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s1
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v51, v49
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v48
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v52, v50
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v25.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v9.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v12.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v27.l
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v26.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v29.l, v13.l, s3
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v49, v48
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v51
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v28.l, v12.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v50
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v11.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v48
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v10.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v49, v49
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v24
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v25.l
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v24.l, s3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v49
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v27.l, v11.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v26.l, v10.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v24.l, v8.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v23
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v49, v49
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v48
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v8.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v23.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v6
+; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v23.l, v7.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v25.l, v9.l, s1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v54.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.h, v31.l, v26.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v54.h, v28.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v26.l, v27.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v36
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v25
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v27.h
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v27, v54
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v28, v28
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v27.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v54.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.h, v36.l, v25.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v54.h, v27.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v25.l, v26.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v33
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v24
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v26.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v26, v54
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v27, v27
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v26.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v22
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v26, v24
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v7.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v22.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v10.l, v8.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v22.l, v6.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v54.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.h, v33.l, v24.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v54.h, v26.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v24.l, v25.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v23
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v25.h
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v25, v54
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v26, v26
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v25.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v22
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.l, v23.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v54.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v54.h, v25.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v23.l, v7.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v6
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v7.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v24, v24
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v7, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v7.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.l, v22.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v54.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v50.l, v54.h, v7.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v22.l, v6.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v21
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v54
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v22, v22
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v6.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v20
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.l, v21.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v54.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.l, v54.h, v6.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v21.l, v5.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v5.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v50
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v5.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v19
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.l, v20.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v54.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v54.h, v5.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v20.l, v4.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v39
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v4.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.l, v19.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v54.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v54.h, v4.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v19.l, v3.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v21
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v9.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v24, v23
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v6.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v21.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v7.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v11.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v8.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v24, v23
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v21.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v6.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v10.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v9.l, v8.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v24, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.l, v11.l, v7.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v21.l, v5.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v20
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v7.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v20.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v6.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v19
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v20.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s0, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v19.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v11
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v4.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v19.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v10.l, v6.l, s3
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v20
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v7.l, v5.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v10, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v11
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v19, v19
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v18.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v7, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v17
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v32
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v3.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v17
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.l, v18.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v54.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.l, v54.h, v3.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v18.l, v2.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v2.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v4, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v35
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v2.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v17.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v54.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.l, v54.h, v2.h, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v16
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v18.l, v2.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v17.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v16.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v7, v7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v5.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v8.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v17.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v16.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v18
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v6.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v17, v16
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v19, v18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v20
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v9.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v8.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v17
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v2.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v11
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v16
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v1.l
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s6, s7
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s5, s8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.l, v8.l, v0.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.l, v7.l, v1.l, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.l, v5.l, v2.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.l, v9.l, v3.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v6.l, v4.l, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v39 :: v_dual_mov_b32 v1, v38
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v37 :: v_dual_mov_b32 v3, v36
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v17.l, v1.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v3, v3
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v38
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v1.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v49
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v16.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v54.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.l, v54.h, v1.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v16.l, v0.h, s2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v52
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v54
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v53.l, v54.h, v0.h, s0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v53
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v34 :: v_dual_mov_b32 v5, v33
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v32 :: v_dual_mov_b32 v7, v31
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v30 :: v_dual_mov_b32 v9, v29
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, v28 :: v_dual_mov_b32 v11, v35
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v32bf16:
@@ -11081,578 +11453,665 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: scratch_load_b32 v55, off, s32
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v53, v15 :: v_dual_mov_b32 v48, v13
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v31, v10
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v33, v8
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v54.l, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v51, v14 :: v_dual_and_b32 v8, 0xffff0000, v53
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v34, v11 :: v_dual_mov_b32 v37, v12
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v36, v9
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v51
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v30
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v24
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v23
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v22
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v48
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v19
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v18
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v17
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v16
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v66.l, v54.l
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v55
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v53.h, v55.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v54.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v55.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX12-TRUE16-NEXT: scratch_load_b32 v31, off, s32
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v10
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v29
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v26
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v13
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v49, v49
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v36, v36
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v50, v50
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v67, v67
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.l, v10.h, v26.h, s9
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s43, v98, v98
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff0000, v12
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v30
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v69, v69
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v67.l, v26.h, v36.l, s10
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v98.l, v36.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v37, v37
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v28
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v50.l, v4.h, v20.h, s21
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v132.l, v67.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v98
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v34, v34
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v12.h, v28.h, s5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v36.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v132
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v38, v38
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v28
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v25
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s21, v98, v132
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v65, v65
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v65.l, v28.h, v34.l, s6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v21
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v52, v52
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v67.l, v67.l, v36.l, s21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v29
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v20
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v68, v68
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v98.l, v67.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s44, v99, v99
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v70, v70
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v13
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v81, v81
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v98
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v14
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v81.l, v20.h, v50.l, s22
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v35, v35
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v23
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s21, 0, v98
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v33, v33
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v22
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v13.h, v29.h, s3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v64, v64
+; GFX12-TRUE16-NEXT: s_and_b32 s5, s21, s5
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v8.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v51.h, v30.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v29
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v37
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v30.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.h, v67.l, v36.l, s5
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v64.l, v29.h, v33.l, s4
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v66, v66
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v71, v71
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.l, v2.h, v18.h, s25
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v51, v51
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v15
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v14
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v85, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.l, v9.h, v25.h, s11
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v14.h, v30.h, s1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v16
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v55, v55
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v68.l, v25.h, v37.l, s12
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v99.l, v37.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v37.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v55.l, v30.h, v32.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v96, v96
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v133.l, v68.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v99
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v29.l, s43
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v96.l, v34.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v130.l, v65.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v133, 16, v133
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v24
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v15
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v85, v85
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v86, v86
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s22, v99, v133
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v86.l, v32.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v128.l, v55.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s44
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v68.l, v68.l, v37.l, s22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v130
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v18
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v54, v54
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v87, v87
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v99.l, v68.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.l, v0.h, v16.h, s29
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v87.l, v33.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v129.l, v64.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v99
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v128
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v30
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.l, v5.h, v21.h, s19
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v118.l, v13.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s22, 0, v99
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s19, v96, v130
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v96.l, v29.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v82, v82
+; GFX12-TRUE16-NEXT: s_and_b32 s6, s22, s6
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v85.l, v16.h, v54.l, s40
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v8.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v48.h, v29.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v28
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v34
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v29.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.h, v68.l, v37.l, s6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v27
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v87
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v97, v97
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v118
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v48, v48
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v48.l, v6.h, v22.h, s17
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s17, v86, v128
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v129
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v65.l, v65.l, v34.l, s19
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v83, v83
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v71.l, v22.h, v48.l, s18
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v102.l, v48.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0, v48.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v83.l, v18.h, v52.l, s26
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v116.l, v54.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v144.l, v71.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v102, 16, v102
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s18, v87, v128
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v87.l, v85.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v19
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v144, 16, v144
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s43, v118, v96
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v96.l, v65.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v8
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v114.l, v52.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s25, v102, v144
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v86.l, v83.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v116
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v87
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v64.l, v64.l, v33.l, s18
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v71.l, v71.l, v48.l, s25
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v80, v80
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v80.l, v21.h, v49.l, s20
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v53, v53
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v102.l, v71.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v30.l, s41
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v114
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s41, v116, v87
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v102, 16, v102
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v55.l, v55.l, v32.l, s17
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v87.l, v64.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s19, 0, v96
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.l, v8.h, v24.h, s13
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s25, 0, v102
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s29, v114, v86
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v86.l, v55.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v87
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v12
+; GFX12-TRUE16-NEXT: s_and_b32 s9, s25, s9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v69.l, v24.h, v38.l, s14
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v8.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v37.h, v28.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v27
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v31
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v28.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v8.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v34.h, v27.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v26
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v36
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v27.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v8.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v31.h, v26.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v25
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v33
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v26.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v8.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v36.h, v25.h, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v25.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v71.l, v48.l, s9
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s18, 0, v87
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s45, v100, v100
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v100.l, v38.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v39, v39
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v103.l, v49.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v112.l, v50.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s42
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v134.l, v69.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v11.h, v27.h, s7
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v145.l, v80.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v146.l, v81.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s17, 0, v86
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.l, v3.h, v19.h, s23
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v66.l, v27.h, v35.l, s8
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v97.l, v35.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v117.l, v14.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v100
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v103
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v131.l, v66.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v97
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v112
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v128.l, v30.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v134, 16, v134
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v131
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v145, 16, v145
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v146, 16, v146
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v84, 0xffff0000, v17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v53.l, v1.h, v17.h, s27
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s20, v97, v131
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v82.l, v19.h, v51.l, s24
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v117
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v128
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s23, v100, v134
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v66.l, v66.l, v35.l, s20
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s26, v103, v145
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s27, v112, v146
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v84, v84
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v113.l, v51.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v97.l, v66.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v147.l, v82.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s42, v117, v128
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v69.l, v69.l, v38.l, s23
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v80.l, v80.l, v49.l, s26
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v97
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v81.l, v81.l, v50.l, s27
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v32.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v84.l, v17.h, v53.l, s28
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v113
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s20, 0, v97
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v147, 16, v147
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s42
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v100.l, v69.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v103.l, v80.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v112.l, v81.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s46, v101, v101
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v28.l, s45
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v115.l, v53.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v129.l, v84.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s28, v113, v147
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v117.l, v30.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v100
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v103
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v112
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s17, s1
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v33.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s46
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v8.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v33.h, v24.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v7
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v24.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v32, v32
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v32.l, v54.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v55.l, v32.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.l, v7.h, v23.h, s15
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v38.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0, v49.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0, v50.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v115
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v129, 16, v129
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v82.l, v82.l, v51.l, s28
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v117
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s23, 0, v100
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s26, 0, v103
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s27, 0, v112
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s18, s2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v119.l, v12.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v130.l, v28.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v7.h, v23.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v23.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v35, v35
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v32
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v32.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v32
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v64.l, v33.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v70.l, v23.h, v39.l, s16
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0, v14.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s40, v115, v129
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v113.l, v82.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s42, 0, v117
+; GFX12-TRUE16-NEXT: s_and_b32 s7, s23, s7
+; GFX12-TRUE16-NEXT: s_and_b32 s10, s26, s10
+; GFX12-TRUE16-NEXT: s_and_b32 s11, s27, s11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v50.h, v32.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v6.h, v22.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v21
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v35, v35
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v22.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v32
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v32.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v32
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.h, v69.l, v38.l, s7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.h, v80.l, v49.l, s10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.h, v81.l, v50.l, s11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v130
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v119
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v83.l, v83.l, v52.l, s29
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v84.l, v84.l, v53.l, s40
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v113
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v11
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0, v51.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v114.l, v83.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v115.l, v84.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s28, 0, v113
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s43
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0, v52.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v114
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v115
+; GFX12-TRUE16-NEXT: s_and_b32 s12, s28, s12
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0, v53.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v34.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s29, 0, v114
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s40, 0, v115
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v35.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v101.l, v39.l
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s19, s3
+; GFX12-TRUE16-NEXT: s_and_b32 s13, s29, s13
+; GFX12-TRUE16-NEXT: s_and_b32 s14, s40, s14
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.h, v32.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v5.h, v21.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v20
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v35, v35
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v35.l, v54.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v21.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v32
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v32.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v32
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v65.l, v34.l, s3
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s20, s4
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v135.l, v70.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v32.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v4.h, v20.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.h, v66.l, v35.l, s4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v101
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v85.l, v85.l, v54.l, s41
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v39.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v135, 16, v135
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0, v54.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v116.l, v85.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s24, v101, v135
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v116
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v70.l, v70.l, v39.l, s24
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.h, v20.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v38, v38
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v38.l, v54.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v35
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.h, v35.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v35
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v49, v49
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s41, 0, v116
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v101.l, v70.l
+; GFX12-TRUE16-NEXT: s_and_b32 s15, s41, s15
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v101
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.h, v35.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v3.h, v19.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.h, v19.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v49, v49
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v49.l, v54.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v38
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.h, v38.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v38
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v52, v52
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.h, v85.l, v54.l, s15
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v31
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v87, 0xffff0000, v31
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v86.l, v15.h, v31.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s24, 0, v101
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v96, v96
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v97.l, v15.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v87, v87
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v96.l, v86.l
+; GFX12-TRUE16-NEXT: s_and_b32 s8, s24, s8
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.l, v31.l, v15.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v97
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v87.l, v31.h, v86.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.h, v38.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v2.h, v18.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.h, v70.l, v39.l, s8
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v99.l, v31.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v98.l, v87.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.h, v18.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v52, v52
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v52.l, v54.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v49
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.h, v49.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v49
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.h, v49.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v1.h, v17.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v99
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v98
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.h, v17.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v64, v64
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v64.l, v54.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v52
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.h, v52.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v52
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.h, v52.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v0.h, v16.h, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v53
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v64.h, v16.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v54, v64
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v64.h, v64.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v55
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v64
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v66.h, v53.l, v55.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v97, v99
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v96, v98
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.l, v31.l, v15.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v15.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v87.l, v86.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v86.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v37.l, v31.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.l, v32.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v36
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v82.l, v51.l, s12
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v51.l, v29.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v37
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.h, v83.l, v52.l, s13
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v38
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.h, v84.l, v53.l, s14
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v53.h, v64.h, v54.h, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v55.l, v66.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v51
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v30
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v66.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v66, v54
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v64, v64
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v66.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v29
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.h, v51.l, v30.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v51.l, v54.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v31.l, v15.l, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s42, s16
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v54.h, v66.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v30.l, v51.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v48
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v51.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v55, v55
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v51, v54
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v30.l, v14.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v50, v49
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v27
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v51.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v30.l, v54.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v27.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v9
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v49, v49
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v26
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v51
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v51.l, v11.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v26.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v49, v49
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.h, v48.l, v29.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v28
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v48
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v48.l, v28.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v53.l, v10.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v50, v50
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v25
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v49.l, v27.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v32.l, v86.l, s1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v52.l, v26.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v25.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v50, v50
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v13.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v52
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v53
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s1
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v51, v49
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v48
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v52, v50
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v48.l, v25.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v49.l, v9.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v12.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v50.l, v27.l
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v51.l, v26.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v54.h, v51.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v29.l, v30.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v37
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v30.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v48, v48
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v30, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v30.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v29.l, v54.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.h, v37.l, v28.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v29.l, v13.l, s3
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v49, v48
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v51
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v28.l, v12.l, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v50
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v11.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v48
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v10.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v49, v49
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v24
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v49.l, v25.l
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX12-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v24.l, s3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v49
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v54.h, v30.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v28.l, v29.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v34
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v27
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v29.h
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v29, v54
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v30, v30
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v29.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v54.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.h, v34.l, v27.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v27.l, v11.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v26.l, v10.l, s1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v24.l, v8.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v23
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v49, v49
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v48
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v9.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v10.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v8.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v23.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v6
+; GFX12-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v23.l, v7.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v54.h, v29.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v27.l, v28.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v31
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v26
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v28.h
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v28, v54
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v29, v29
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v28.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v25.l, v9.l, s1
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v54.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.h, v31.l, v26.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v54.h, v28.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v26.l, v27.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v36
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v25
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v27.h
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v27, v54
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v28, v28
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v27.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v54.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v22
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v26, v24
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v11.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v7.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.h, v36.l, v25.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v54.h, v27.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v25.l, v26.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v33
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v24
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v26.h
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v26, v54
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v27, v27
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v22.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v10.l, v8.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v26.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v22.l, v6.l, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v54.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.h, v33.l, v24.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v54.h, v26.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v24.l, v25.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v23
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v25.h
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v25, v54
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v26, v26
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v25.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v22
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v21
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v9.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v24, v23
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v10.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v6.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.l, v23.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v54.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v21.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v7.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v11.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v8.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v24, v23
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v21.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v5.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v6.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v7.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v10.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v54.h, v25.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v23.l, v7.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v6
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v7.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v24, v24
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v7, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v7.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.l, v22.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v54.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v9.l, v8.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v24, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v22
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v50.l, v54.h, v7.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v22.l, v6.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v21
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v54
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v22, v22
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v6.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v20
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.l, v11.l, v7.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v21.l, v5.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v20
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v7.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.l, v21.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v54.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.l, v54.h, v6.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v21.l, v5.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v5.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v50
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v54
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v20.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v6.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v19
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v5.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v19
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v20.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s0, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v19.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v11
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v5.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v4.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.l, v20.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v54.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v19.l, v3.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v54.h, v5.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v20.l, v4.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v39
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v4.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v18
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.l, v19.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v54.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v10.l, v6.l, s3
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v20
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v9.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v54.h, v4.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v19.l, v3.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.h
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v7.l, v5.l, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v10, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v11
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v19, v19
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v18
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v18.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v7, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v17
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v32
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v3.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v17
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.l, v18.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v54.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.l, v54.h, v3.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v18.l, v2.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v2.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v4, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v35
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v2.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v16
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v16
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v18.l, v2.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v17.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v16.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v7, v7
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v5.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v17.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v54.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v8.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v17.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v16.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v18
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v7.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v8.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v0.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v6.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v10
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v3.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v17, v16
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v19, v18
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v20
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v9.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v5.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v7.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v8.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v17
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v2.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v11
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v16
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v1.l
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s6, s7
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s5, s8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.l, v54.h, v2.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v17.l, v1.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v3, v3
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v38
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v49
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.l, v8.l, v0.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.l, v7.l, v1.l, s4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.l, v5.l, v2.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.l, v9.l, v3.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v6.l, v4.l, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v39 :: v_dual_mov_b32 v1, v38
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, v37 :: v_dual_mov_b32 v3, v36
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v16.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v54.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.l, v54.h, v1.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v16.l, v0.h, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v52
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v54
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v53.l, v54.h, v0.h, s0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v53
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v34 :: v_dual_mov_b32 v5, v33
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v6, v32 :: v_dual_mov_b32 v7, v31
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v8, v30 :: v_dual_mov_b32 v9, v29
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v10, v28 :: v_dual_mov_b32 v11, v35
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v32bf16:
@@ -12412,25 +12871,31 @@ define bfloat @v_maximumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
; GFX11-TRUE16-LABEL: v_maximumnum_bf16_no_ieee:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v0.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v1.h, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.h, v1.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v1.h, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_bf16_no_ieee:
@@ -12464,28 +12929,34 @@ define bfloat @v_maximumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v0.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.h
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v1.h, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_bf16_no_ieee:
@@ -12696,38 +13167,52 @@ define <2 x bfloat> @v_maximumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v1.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.h, v2.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v2.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.h, v2.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.l, v4.h, s2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.h, v4.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v4.h, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v2.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v2.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v2bf16_no_ieee:
@@ -12781,44 +13266,56 @@ define <2 x bfloat> @v_maximumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.h, v2.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v2.h
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v2.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.h, v2.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.l, v4.h, s2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.h
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v6
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.h, v4.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v2.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v4.h, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v2.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v2bf16_no_ieee:
@@ -13119,54 +13616,67 @@ define <3 x bfloat> @v_maximumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v8, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.h, v2.h, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v2.h, v4.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v4.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v5.h, v4.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.l, v1.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v1.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.h, v1.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.l, v0.h, s2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v4.h, v0.h, s0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v4.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v9
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v10, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v4.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s3, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s4, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v3bf16_no_ieee:
@@ -13235,62 +13745,72 @@ define <3 x bfloat> @v_maximumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v8, v8
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.h, v2.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v2.h, v4.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.h
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v4.l, s1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v8
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v9
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v10, v11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v4.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v4.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v5.h, v4.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.l, v1.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v1, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s3, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.h, v1.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.l, v0.h, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s4, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v4.h, v0.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v3bf16_no_ieee:
@@ -13687,70 +14207,77 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v1.h, v3.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.h, v6.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v6.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v6.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v6.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v0.h, v2.h, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v2.h, v6.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v6.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v10, v10
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v1.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.h, v6.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v3.l, v7.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v7.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v7, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v7.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v6.h, v7.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.l, v0.h, s2
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.h, v0.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v1 :: v_dual_mov_b32 v1, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v4.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v4.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v1.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v10, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v9
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v12, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v4.l, s5
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v10
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v6.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v4bf16_no_ieee:
@@ -13843,81 +14370,86 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v1.h, v3.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.h, v6.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v6.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v6.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v6.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v0.h, v2.h, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v2.h, v6.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.h
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v5
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v6.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v10, v10
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v4.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v4.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v6.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v9
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v10, v8
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v9
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v12, v13
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v1.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.h, v6.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v3.l, v7.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v7.h
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v7.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v4.l, s5
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v10
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v6.h, v7.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.l, v0.h, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v0, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v6.l, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.h, v0.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v1 :: v_dual_mov_b32 v1, v4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v4bf16_no_ieee:
diff --git a/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
index 0a794a3ac49b1..bbb026c17c811 100644
--- a/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
@@ -109,25 +109,31 @@ define bfloat @v_minimumnum_bf16(bfloat %x, bfloat %y) {
; GFX11-TRUE16-LABEL: v_minimumnum_bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v0.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v1.h, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.h, v1.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v1.h, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_bf16:
@@ -161,28 +167,34 @@ define bfloat @v_minimumnum_bf16(bfloat %x, bfloat %y) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v0.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.h
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v1.h, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_bf16:
@@ -293,18 +305,19 @@ define bfloat @v_minimumnum_bf16_nnan(bfloat %x, bfloat %y) {
; GFX11-TRUE16-LABEL: v_minimumnum_bf16_nnan:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v0.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v2
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_bf16_nnan:
@@ -330,20 +343,20 @@ define bfloat @v_minimumnum_bf16_nnan(bfloat %x, bfloat %y) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.l, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v2
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_bf16_nnan:
@@ -552,38 +565,52 @@ define <2 x bfloat> @v_minimumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v1.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.h, v2.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v2.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.h, v2.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.l, v4.h, s2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.h, v4.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v4.h, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v2.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v2.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v2bf16:
@@ -637,44 +664,56 @@ define <2 x bfloat> @v_minimumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.h, v2.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v2.h
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v2.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.h, v2.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.l, v4.h, s2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.h
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v6
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.h, v4.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v2.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v4.h, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v2.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v2bf16:
@@ -859,24 +898,29 @@ define <2 x bfloat> @v_minimumnum_v2bf16_nnan(<2 x bfloat> %x, <2 x bfloat> %y)
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v4, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v0.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.h, v0.h, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.h, v0.h, s0
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.h, v0.h, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v4
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v0.h, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v2bf16_nnan:
@@ -917,27 +961,31 @@ define <2 x bfloat> @v_minimumnum_v2bf16_nnan(<2 x bfloat> %x, <2 x bfloat> %y)
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v4, v3
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v0.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.h, v0.h, s1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.h, v0.h, s0
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v4
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.h, v0.h, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v0.h, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v2bf16_nnan:
@@ -1229,54 +1277,67 @@ define <3 x bfloat> @v_minimumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v8, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.h, v2.h, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v2.h, v4.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v4.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v5.h, v4.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.l, v1.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v1.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.h, v1.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.l, v0.h, s2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v4.h, v0.h, s0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v4.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v9
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v10, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s3, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s4, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v3bf16:
@@ -1345,62 +1406,72 @@ define <3 x bfloat> @v_minimumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v8, v8
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.h, v2.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v2.h, v4.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.h
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v4.l, s1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v8
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v9
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v10, v11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v4.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s3, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v5.h, v4.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.l, v1.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.h, v1.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.l, v0.h, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s4, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v4.h, v0.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v3bf16:
@@ -1648,33 +1719,40 @@ define <3 x bfloat> @v_minimumnum_v3bf16_nnan(<3 x bfloat> %x, <3 x bfloat> %y)
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v6
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v9, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v2.h, v0.h, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v5, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.h, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.h, v0.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.h, v0.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.h, v0.h, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v7
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s3, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s4, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, s1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v3bf16_nnan:
@@ -1723,37 +1801,43 @@ define <3 x bfloat> @v_minimumnum_v3bf16_nnan(<3 x bfloat> %x, <3 x bfloat> %y)
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v0
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v6
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v9, v8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v2.h, v0.h, s1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v5, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.h, v1.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v5, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v7
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s3, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.h, v0.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.h, v0.h, s1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s4, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.h, v0.h, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, s1
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v3bf16_nnan:
@@ -2135,70 +2219,77 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v1.h, v3.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.h, v6.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v6.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v6.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v0.h, v2.h, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v2.h, v6.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v6.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v10, v10
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v1.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.h, v6.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v3.l, v7.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v7.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v7, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v7.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v6.h, v7.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.l, v0.h, s2
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.h, v0.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v1 :: v_dual_mov_b32 v1, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v4.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v4.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v1.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v10, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v9
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v12, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v4.l, s5
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v10
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v6.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v4bf16:
@@ -2291,81 +2382,86 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) {
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v1.h, v3.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.h, v6.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v6.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v6.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v0.h, v2.h, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v2.h, v6.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v5
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v6.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v10, v10
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v4.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v4.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v6.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v9
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v10, v8
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v9
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v12, v13
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v1.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.h, v6.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v3.l, v7.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v7.h
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v7.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v4.l, s5
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v10
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v6.h, v7.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.l, v0.h, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v6.l, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.h, v0.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v1 :: v_dual_mov_b32 v1, v4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v4bf16:
@@ -2692,42 +2788,45 @@ define <4 x bfloat> @v_minimumnum_v4bf16_nnan(<4 x bfloat> %x, <4 x bfloat> %y)
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v6
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v9, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v11, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.h, v1.h, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v6, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v0.h, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v1.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v7
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v9
; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.h, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.h, v1.h, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v5, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.h, v1.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.h, v0.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.h, v0.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.h, v0.h, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s5, s6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v0.h, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v1.h, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v4bf16_nnan:
@@ -2792,47 +2891,49 @@ define <4 x bfloat> @v_minimumnum_v4bf16_nnan(<4 x bfloat> %x, <4 x bfloat> %y)
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v0
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v6
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v9, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v11, v10
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.h, v1.h, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v6, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.h
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.h, v1.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.h, v1.h, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v5, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.h, v1.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v5, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v0.h, s2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v1.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v7
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v8
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v9
; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.h
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.h, v0.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.h, v0.h, s1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s5, s6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.h, v0.h, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v0.h, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v1.h, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v4bf16_nnan:
@@ -3386,101 +3487,112 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, 0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v8.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v2.h, v5.h, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v5.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v11, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v5.h, v8.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v8.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v8.h, s0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v7, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v1.h, v4.h, s1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v4.h, v8.h, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.h, v8.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8.h
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v5.h, v6.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v12, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v1.h, v4.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v13, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v14, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v8.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v15, v15
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v10, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v0.h, v3.h, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v4.h, v8.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v16, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v3.h, v9.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v9.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v17, v17
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v11.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v5.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v12, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v16
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v8.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s6
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v15, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v7.l, v6.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v4.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v11.l, v9.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v10.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v14, v13
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v16, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v12
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v7
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.h, v8.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v0.h, v3.h, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v3.h, v8.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v8.h
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v9.h, v8.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v2.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v9.h, v8.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v5.l, v10.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v10.h
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v10, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v10.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v8.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v8.h, v10.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v4.l, v1.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v1.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v8.h, v1.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v3.l, v0.h, s2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v7
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v8.h, v0.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s2, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s0, s1
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v11
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v2.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v7
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v12
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v1.l
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v10.l, v8.l, s5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v9.l, s6
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, s3
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v2.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v6bf16:
@@ -3610,118 +3722,124 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) {
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, 0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v8.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v5
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v2.h, v5.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v5.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v11, v11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v5.h, v8.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v8.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v8.h, s0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v7, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v5.h, v6.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v12, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v8
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v1.h, v4.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v13, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v14, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v3
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v8.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v15, v15
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v10, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v0.h, v3.h, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v1.h, v4.h, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v4.h, v8.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v7
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.h, v8.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v7
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.h, v8.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v0.h, v3.h, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v3.h, v8.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v8.h
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v9
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v9.h, v8.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v2.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v9.h, v8.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v5.l, v10.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v10.h
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v10, v8
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v10.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v8.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v4.h, v8.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v3.h, v9.l, s2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v10.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v9.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v17, v17
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v7.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v11.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v5.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v12, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v16
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v14
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v8.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s6
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v15, v12
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v8.h, v10.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v4.l, v1.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v7.l, v6.l, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v4.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v11.l, v9.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v0.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v10.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v7
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v14, v13
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v16, v15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v12
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v7
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v9.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-TRUE16-NEXT: s_and_b32 s5, s2, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s6, s0, s1
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v11
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v2.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v7
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v12
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v1.l
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v8.h, v1.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v3.l, v0.h, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v7
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v10.l, v8.l, s5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v9.l, s6
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, s3
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v8.h, v0.h, s0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v2.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v6bf16:
@@ -4500,132 +4618,154 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v14, v14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v3.h, v7.h, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v11.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v7.h, v11.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v11.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v11.h, s0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v2.h, v6.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v6.h, v11.h, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v11.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v9.h, v11.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v9
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v9.h, v11.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v1.h, v5.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v5.h, v11.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v12, v12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v11.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v10.h, v11.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v3.h, v7.h, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.h, v6.h, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v7.h, v8.l, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v9.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v6.h, v9.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v10.h, v11.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v0.h, v4.h, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v4.h, v11.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v11.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v10.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v1.h, v5.h, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v11.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v5.h, v12.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v8.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v9.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v17, v14
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v15, v16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v13.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v8.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v9.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v10.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v11.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v15, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v19
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v12.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v0.h, v4.h, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v15
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v13.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v4.h, v14.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v14.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v7.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v15.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v18
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s1, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v17
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v12.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v10.l, v8.l, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s4, s3
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v16, v17
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v11.l, v9.l, s3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v13.l, v12.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v15.l, v14.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v15, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v6.l, v9.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v12.h, v11.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v11.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v3.l, v7.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v12.h, v11.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v7.l, v13.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v6
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v13.h
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v13, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v12, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v11.h, v13.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.l, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v11.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v11.h, v13.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v6.l, v2.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v2.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v11.h, v2.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v11.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v11.h, v2.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v5.l, v1.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v11
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v11.h, v1.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v11.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v11.h, v1.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v4.l, v0.h, s2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v11.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v11
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v11.h, v0.h, s0
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v3 :: v_dual_mov_b32 v3, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v9.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v7.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v13, v12
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v16, v15
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.l, v9.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v6.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v7.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v4.l
+; GFX11-TRUE16-NEXT: s_and_b32 s7, s2, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v12
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v9.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v1.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v11
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, s6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v9.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v1.l, s3
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s4, s5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v14.l, s7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.l, v0.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v6.l, v3.l, s0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v2 :: v_dual_mov_b32 v2, v7
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v8bf16:
@@ -4784,154 +4924,175 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v7
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v7
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v1
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v14, v14
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v3.h, v7.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v11.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v7.h, v11.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v11.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v11.h, s0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v6
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v2.h, v6.h, s1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v6.h, v11.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v11.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v9
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v9.h, v11.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v9
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v9.h, v11.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v1.h, v5.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v5.h, v11.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v12, v12
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v11.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v10
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v10.h, v11.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v11.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v10.h, v11.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v0.h, v4.h, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v4.h, v11.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v11.h
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v12
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v3.h, v7.h, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.h, v6.h, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v7.h, v8.l, s1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v9.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v12.h, v11.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v6.h, v9.l, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v11.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v12
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v8.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v10.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v1.h, v5.h, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v11.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v5.h, v12.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v8.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v9.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v17, v14
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v15, v16
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v12.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v13.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v8.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v9.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v10.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v11.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v15, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v18
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v19
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v12.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v14
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v0.h, v4.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v15
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v13.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v4.h, v14.l, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v14.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v7.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v15.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v18
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s1, s2
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v17
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v12.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v19
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v3.l, v7.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v3.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v12.h, v11.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v7.l, v13.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v6
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v13.h
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v13, v11
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v12, v12
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v11.h, v13.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v11
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.l, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v11.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v10.l, v8.l, s2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s4, s3
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v16, v17
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v11.h, v13.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v6.l, v2.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v2.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v11.l, v9.l, s3
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v13.l, v12.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v11.h, v2.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v15.l, v14.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v11.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v11.h, v2.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v5.l, v1.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v11
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v15, v15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v11.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v9
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v6.l, v9.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v11.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v12
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v9.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v4.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v0.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v7.l, v3.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v13, v12
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v16, v15
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v10
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.l, v9.l, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v6.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v14.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v7.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v4.l
+; GFX12-TRUE16-NEXT: s_and_b32 s7, s2, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v12
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v9.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v11
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s3, s6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v11.h, v1.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v4.l, v0.h, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v10
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v11
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v11.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v11
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v9.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v1.l, s3
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s4, s5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v14.l, s7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v11.h, v0.h, s0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v3 :: v_dual_mov_b32 v3, v8
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.l, v0.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v6.l, v3.l, s0
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v2 :: v_dual_mov_b32 v2, v7
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v8bf16:
@@ -6346,258 +6507,310 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
; GFX11-TRUE16-LABEL: v_minimumnum_v16bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v16, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v16
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v17, v17
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v23.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v10
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, v7 :: v_dual_mov_b32 v17, v6
; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v15
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v16.h, v15.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v14
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v16
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v17
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v16.h, v15.h, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v23.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v26, v26
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v15.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v7
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.h, v23.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v6.h, v14.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.h, v14.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v18, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v23.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.h, v17.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v17
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v17.h, v14.h, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v6.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v15.h, v6.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.h, v17.h, v23.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v5.h, v13.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.h, v13.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v19, v19
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v23.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.h, v18.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v18
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.h, v18.h, v23.h, s0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v4.h, v12.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.h, v12.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v20, v20
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v23.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v19
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.h, v19.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v19
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.h, v19.h, v23.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v3.h, v11.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.h, v11.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v23.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v20
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.h, v20.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v20
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v18.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v6.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v7.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v14.h, v18.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.h, v20.h, v23.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v2.h, v10.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.h, v10.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v23.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v21
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.h, v21.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v21
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.h, v21.h, v23.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v1.h, v9.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.h, v9.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v24, v24
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v23.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v22
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.h, v22.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v22
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v18.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v19.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v20, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v5.h, v13.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v21, v23
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v13.h, v20.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v20.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v18.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v12
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v21.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v22
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v19.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v4.h, v12.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v26
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v12.h, v22.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v24, v25
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v22.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v27
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v23.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v20.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v21.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.l, v6.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v19.l, v18.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v24, v25
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v27, v27
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v26
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v23.l, v22.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v3.h, v11.h, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v10
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v18
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v6.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v11.h, v7.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v2.h, v10.h, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v7.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v24
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v18.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v10.h, v19.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v24
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v23.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v1.h, v9.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v25, v28
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v29
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v30
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v20.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v9.h, v24.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v27
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v7.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v26, v28
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v24.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v25.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v22.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v18.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v19.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v23.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v26, v27
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.h, v21.l, v20.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.h, v6.l, v22.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v29
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v25.l, v24.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v28
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v20
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v21, v21
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v8
; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.h, v22.h, v23.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v0.h, v8.h, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.h, v8.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v23.h
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v24
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.h, v24.h, v23.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v14
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v24
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v16.l, v15.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v23.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v24.h, v23.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v15.l, v16.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v6
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v16.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v25, v25
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v16, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v16.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.l, v14.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v23.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v23.h, v16.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v14.l, v6.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v13
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v23
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v16, v16
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v6.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.l, v13.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v23.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v23.h, v6.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v13.l, v5.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v12
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v5.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v23
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v13, v13
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v5.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.l, v12.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v23.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v23.h, v5.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v12.l, v4.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v17
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v4.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.l, v11.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v23.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v23.h, v4.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v11.l, v3.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v18
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v3.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.l, v10.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v23.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v23.h, v3.h, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v19.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v10.l, v2.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v2.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v4, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v19
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v2.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v9.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v23.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v23.h, v2.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v9.l, v1.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v3, v3
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v20
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v1.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v21
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v8.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v23.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v23.h, v1.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v8.l, v0.h, s2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v22
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v23
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v23.h, v0.h, s0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v20
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v0.h, v8.h, s3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v16
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.h, v18.l, v7.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v8.h, v20.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.h, v23.l, v19.l, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v20.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v15.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v17
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v24.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v16.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v14
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v6.l, v24.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v22, v23
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v16.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v17.l, v14.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v20.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v23
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v13
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v22, v21
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v6.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v13.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v23
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v16.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v17
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v15.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v20.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v22, v21
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v17
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v14.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v7.l, v20.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v22, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v4
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v15.l, v16.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v12
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v13.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v12.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v6.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v17, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s0, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v11.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v21
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v5.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v14.l, v6.l, s3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v11.l
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v13.l, v5.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v17, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v21
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v22, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v3.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v10.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v9
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v22, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v10.l, v2.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v8.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v5.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v12.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v13, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v17
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v10.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v13, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v17, v14
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v22, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v23
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v5.l, v2.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v11.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v8.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v12.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v14
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v5.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v2.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v13
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v14
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v1.l
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s6, s7
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s5, s8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v0.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v9.l, v1.l, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v12.l, v2.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v11.l, v3.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v10.l, v4.l, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v15
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v20
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v16 :: v_dual_mov_b32 v4, v19
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v18
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v16bf16:
@@ -6923,299 +7136,356 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v16, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v6
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v14
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v13
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v16
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v11
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v17, v17
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v23.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v10
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v16, v7 :: v_dual_mov_b32 v17, v6
; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v15
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v9
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v15
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v14
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v16
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v17
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v16.h, v15.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v16.h, v15.h, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v23.l
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v26, v26
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v15.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v7
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v7
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.h, v23.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v6.h, v14.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v5
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.h, v14.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v18, v18
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v23.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v17
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v17.h, v14.h, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v6.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.h, v17.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v17
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v15.h, v6.l, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.h, v17.h, v23.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v5.h, v13.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.h, v13.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v19, v19
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v23.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v18
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v18.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v6.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v7.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.h, v18.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v18
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.h, v18.h, v23.h, s0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v12
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v4.h, v12.h, s1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.h, v12.h, v23.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v14.h, v18.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v13
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v18.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v19.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v20, v20
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v23.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v19
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.h, v19.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v19
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.h, v19.h, v23.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v3.h, v11.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.h, v11.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v23.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v20
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v20, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.h, v20.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v20
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v5.h, v13.h, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.h, v20.h, v23.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v2.h, v10.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v21, v23
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.h, v10.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v23.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v21
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v13.h, v20.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v7.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v20.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v18.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v12
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v21.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v22
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v19.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.h, v21.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v21
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v4.h, v12.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v26
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v12.h, v22.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v24, v25
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v22.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v27
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v23.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v20.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX12-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s3
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v21.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.h, v21.h, v23.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v1.h, v9.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.h, v9.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v24, v24
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v23.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v22
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.l, v6.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v19.l, v18.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v24, v25
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v27, v27
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v11
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v26
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v23.l, v22.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v3.h, v11.h, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v10
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v18
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v6.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.h, v22.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v22
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.h, v22.h, v23.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v0.h, v8.h, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v16
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v11.h, v7.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v2.h, v10.h, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v7.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v24
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v18.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.h, v8.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v23.h
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v24
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v10.h, v19.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff0000, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v24
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v29.l, v19.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v30.l, v23.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.h, v24.h, v23.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v14
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v24
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v1.h, v9.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v25, v28
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v29
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v30
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v20.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v16.l, v15.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v23.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v9.h, v24.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v27
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v7.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v26, v28
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v24.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v25.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v22.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v18.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v19.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s3
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v29.l, v23.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v26, v27
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v24.h, v23.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v15.l, v16.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v6
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v16.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v25, v25
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v16, v23
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v16.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.l, v14.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v23.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.h, v21.l, v20.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.h, v6.l, v22.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v29
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v25.l, v24.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v28
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v7.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v20
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v21, v21
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v6.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v8
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v19.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v20
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v0.h, v8.h, s3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v16
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v23.h, v16.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v14.l, v6.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v13
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v23
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v16, v16
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v6.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.h, v18.l, v7.l, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v22
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.l, v13.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v23.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v8.h, v20.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v15
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v23.h, v6.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v13.l, v5.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v12
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v5.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v23
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v13, v13
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.h, v23.l, v19.l, s1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v20.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v5.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v11
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v15.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v17
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v24.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.l, v12.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v23.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v16.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v14
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v23.h, v5.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v12.l, v4.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v17
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v23
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v4.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v10
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v6.l, v24.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v22, v23
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v15.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v16.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.l, v11.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v23.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v17.l, v14.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v20.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v23
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v13
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v22, v21
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v14.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v6.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v13.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v23
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v16.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v17
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v15.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v20.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v22, v21
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v13.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v5.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v17
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v16.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v14.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v23.h, v4.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v11.l, v3.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v18
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v23
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v3.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v9
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.l, v10.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v23.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v7.l, v20.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v22, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v4
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v23.h, v3.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v10.l, v2.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v2.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v4, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v19
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v23
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v15.l, v16.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v12
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v3
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v13.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v2.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v8
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v12.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v6.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v17, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v22
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s0, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v11.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v21
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v12.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v5.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v9.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v23.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v23.h, v2.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v9.l, v1.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v3, v3
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v20
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v23
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v21
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v23
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v8.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v23.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v14.l, v6.l, s3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v11.l
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v23.h, v1.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v8.l, v0.h, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v22
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v23
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.h, v23.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v23
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v13.l, v5.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v17, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v21
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v22, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v3.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v10.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v9
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v22, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v8
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v10.l, v2.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v8.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v5.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v12.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v13, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v17
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v9.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v8.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v0.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v10.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v13, v12
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v17, v14
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v22, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v23
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v5.l, v2.l, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v11.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v8.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v12.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v14
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v9.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v5.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v17
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v2.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v13
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v14
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v1.l
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s6, s7
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s5, s8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v23.h, v0.h, s0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v15
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v0.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v9.l, v1.l, s4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v12.l, v2.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v11.l, v3.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v10.l, v4.l, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v15
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, v20 :: v_dual_mov_b32 v3, v16
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v19 :: v_dual_mov_b32 v5, v18
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v16bf16:
@@ -10043,500 +10313,602 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
; GFX11-TRUE16-LABEL: v_minimumnum_v32bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: scratch_load_b32 v55, off, s32
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v53, v15 :: v_dual_mov_b32 v48, v13
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v31, v10
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v33, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v51, v14 :: v_dual_and_b32 v8, 0xffff0000, v53
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v34, v11 :: v_dual_mov_b32 v37, v12
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v36, v9
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v51
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v30
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v24
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v23
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v22
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v48
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v19
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v18
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v17
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v54.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v55
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v53.h, v55.h, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v54.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v55.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v8.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v51.h, v30.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v29
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v37
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v30.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v8.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v48.h, v29.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v28
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v34
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v29.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v8.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v37.h, v28.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v27
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v31
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v28.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v8.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v34.h, v27.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v26
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v36
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v27.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v8.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v31.h, v26.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v25
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v33
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v26.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v8.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v36.h, v25.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v25.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v10
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v29
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v26
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v13
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v49, v49
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v36, v36
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v50, v50
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v67, v67
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.l, v10.h, v26.h, s9
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s43, v98, v98
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff0000, v12
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v30
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v69, v69
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v67.l, v26.h, v36.l, s10
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v98.l, v36.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v37, v37
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v28
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v50.l, v4.h, v20.h, s21
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v132.l, v67.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v98
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v34, v34
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v12.h, v28.h, s5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v36.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v132
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v38, v38
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v28
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v25
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s21, v98, v132
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v65, v65
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v65.l, v28.h, v34.l, s6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v21
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v52, v52
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v67.l, v67.l, v36.l, s21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v29
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v20
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v68, v68
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v98.l, v67.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s44, v99, v99
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v70, v70
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v81, v81
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v98
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v81.l, v20.h, v50.l, s22
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v35, v35
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v23
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s21, 0, v98
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v33, v33
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v13.h, v29.h, s3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v64, v64
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s21, s5
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.h, v67.l, v36.l, s5
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v64.l, v29.h, v33.l, s4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v66, v66
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v71, v71
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.l, v2.h, v18.h, s25
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v51, v51
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v15
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v14
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v85, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.l, v9.h, v25.h, s11
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v8.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v33.h, v24.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v24.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v32, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v54.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v7.h, v23.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v23.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v35, v35
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v32
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v32.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v32
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v50.h, v32.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v6.h, v22.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v21
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v35, v35
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v22.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v32
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v32.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v32
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.h, v32.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v5.h, v21.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v20
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v35, v35
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v54.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v21.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v32
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v32.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v32
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v32.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v4.h, v20.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v14.h, v30.h, s1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v16
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v55, v55
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v68.l, v25.h, v37.l, s12
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v99.l, v37.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v37.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v55.l, v30.h, v32.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v96, v96
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v133.l, v68.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v99
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v29.l, s43
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v130.l, v65.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v133, 16, v133
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v24
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v15
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v85, v85
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v86, v86
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s22, v99, v133
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v128.l, v55.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s44
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v68.l, v68.l, v37.l, s22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v130
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v18
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v54, v54
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v87, v87
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v99.l, v68.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.l, v0.h, v16.h, s29
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v33.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v129.l, v64.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v99
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v128
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v30
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.l, v5.h, v21.h, s19
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v118.l, v13.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s22, 0, v99
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s19, v96, v130
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v29.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v82, v82
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s22, s6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v85.l, v16.h, v54.l, s40
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.h, v68.l, v37.l, s6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v27
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v87
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v97, v97
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v118
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v48, v48
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v48.l, v6.h, v22.h, s17
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s17, v86, v128
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v129
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v65.l, v65.l, v34.l, s19
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v83, v83
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v71.l, v22.h, v48.l, s18
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v102.l, v48.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0x8000, v48.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v83.l, v18.h, v52.l, s26
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v116.l, v54.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.l, v71.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v102, 16, v102
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s18, v87, v128
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v85.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v19
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v144, 16, v144
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s43, v118, v96
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v65.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.l, v52.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s25, v102, v144
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v83.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v116
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v87
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v64.l, v64.l, v33.l, s18
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v71.l, v71.l, v48.l, s25
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v80, v80
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v80.l, v21.h, v49.l, s20
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v53, v53
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v102.l, v71.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v30.l, s41
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v114
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s41, v116, v87
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v102, 16, v102
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v55.l, v55.l, v32.l, s17
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v64.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s19, 0, v96
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.l, v8.h, v24.h, s13
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s25, 0, v102
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s29, v114, v86
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v55.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v87
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v12
+; GFX11-TRUE16-NEXT: s_and_b32 s9, s25, s9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v69.l, v24.h, v38.l, s14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v71.l, v48.l, s9
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s18, 0, v87
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s45, v100, v100
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v100.l, v38.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v39, v39
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v103.l, v49.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v112.l, v50.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s42
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v134.l, v69.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v11.h, v27.h, s7
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v145.l, v80.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v146.l, v81.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s17, 0, v86
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.l, v3.h, v19.h, s23
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v66.l, v27.h, v35.l, s8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.l, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v117.l, v14.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v100
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v103
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.l, v66.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v97
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v112
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v128.l, v30.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v134, 16, v134
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v131
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v145, 16, v145
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v146, 16, v146
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v84, 0xffff0000, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v53.l, v1.h, v17.h, s27
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s20, v97, v131
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v82.l, v19.h, v51.l, s24
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v117
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v128
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s23, v100, v134
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v66.l, v66.l, v35.l, s20
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s26, v103, v145
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s27, v112, v146
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v84, v84
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v113.l, v51.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.l, v66.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v147.l, v82.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s42, v117, v128
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v69.l, v69.l, v38.l, s23
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v80.l, v80.l, v49.l, s26
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v97
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v81.l, v81.l, v50.l, s27
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v32.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v84.l, v17.h, v53.l, s28
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v113
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s20, 0, v97
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v147, 16, v147
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s42
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v100.l, v69.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v103.l, v80.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v112.l, v81.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s46, v101, v101
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v28.l, s45
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v115.l, v53.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v129.l, v84.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s28, v113, v147
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v117.l, v30.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v100
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v103
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v112
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s17, s1
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v33.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s46
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v55.l, v32.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.l, v7.h, v23.h, s15
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v38.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0x8000, v49.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0x8000, v50.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v115
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v129, 16, v129
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v82.l, v82.l, v51.l, s28
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v117
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s23, 0, v100
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s26, 0, v103
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s27, 0, v112
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s18, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v130.l, v28.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v64.l, v33.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v70.l, v23.h, v39.l, s16
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0x8000, v14.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s40, v115, v129
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v113.l, v82.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s42, 0, v117
+; GFX11-TRUE16-NEXT: s_and_b32 s7, s23, s7
+; GFX11-TRUE16-NEXT: s_and_b32 s10, s26, s10
+; GFX11-TRUE16-NEXT: s_and_b32 s11, s27, s11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.h, v69.l, v38.l, s7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.h, v80.l, v49.l, s10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.h, v81.l, v50.l, s11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v130
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v119
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v83.l, v83.l, v52.l, s29
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v84.l, v84.l, v53.l, s40
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v113
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v11
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0x8000, v51.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.l, v83.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v115.l, v84.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s28, 0, v113
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s43
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0x8000, v52.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v114
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v115
+; GFX11-TRUE16-NEXT: s_and_b32 s12, s28, s12
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0x8000, v53.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v34.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s29, 0, v114
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s40, 0, v115
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v101.l, v39.l
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s19, s3
+; GFX11-TRUE16-NEXT: s_and_b32 s13, s29, s13
+; GFX11-TRUE16-NEXT: s_and_b32 s14, s40, s14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v65.l, v34.l, s3
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s20, s4
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v135.l, v70.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.h, v66.l, v35.l, s4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v101
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v85.l, v85.l, v54.l, s41
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v39.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v135, 16, v135
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0x8000, v54.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v116.l, v85.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s24, v101, v135
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.h, v20.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v38, v38
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v54.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v35
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.h, v35.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v35
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.h, v35.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v3.h, v19.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v116
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v70.l, v70.l, v39.l, s24
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.h, v19.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v49, v49
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v54.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v38
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.h, v38.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v38
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.h, v38.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v2.h, v18.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s41, 0, v116
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v101.l, v70.l
+; GFX11-TRUE16-NEXT: s_and_b32 s15, s41, s15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v101
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.h, v85.l, v54.l, s15
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v31
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v87, 0xffff0000, v31
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v86.l, v15.h, v31.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s24, 0, v101
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v96, v96
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.l, v15.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v87, v87
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v86.l
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s24, s8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.l, v31.l, v15.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v97
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v87.l, v31.h, v86.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.h, v70.l, v39.l, s8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v99.l, v31.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v98.l, v87.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v99
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.h, v18.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v52, v52
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v54.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v49
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.h, v49.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v49
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.h, v49.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v1.h, v17.h, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v98
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v97, v99
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.h, v17.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v64, v64
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v54.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v52
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.h, v52.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v52
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.h, v52.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v0.h, v16.h, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v53
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v64.h, v16.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v54.h
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v64
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v64.h, v64.h, v54.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v55
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v64
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v66.h, v53.l, v55.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v53.h, v64.h, v54.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v55.l, v66.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v51
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v30
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v66.h
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v96, v98
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.l, v31.l, v15.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v15.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v87.l, v86.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v86.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v66, v54
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v64, v64
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v66.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v29
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.h, v51.l, v30.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v54.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v54.h, v66.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v30.l, v51.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v48
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v51.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v55, v55
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v51, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v51.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v54.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.h, v48.l, v29.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v28
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v54.h, v51.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v29.l, v30.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v37
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v30.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v48, v48
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v30, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v30.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v54.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.h, v37.l, v28.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v54.h, v30.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v28.l, v29.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v34
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v27
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v29.h
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v29, v54
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v30, v30
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v29.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v54.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.h, v34.l, v27.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v54.h, v29.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v27.l, v28.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v31
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v26
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v28.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v28, v54
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v29, v29
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v28.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v31.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v32.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v36
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v82.l, v51.l, s12
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v29.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v37
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.h, v83.l, v52.l, s13
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v38
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.h, v84.l, v53.l, s14
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v31.l, v15.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s42, s16
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v30.l, v14.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v50, v49
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v27
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v27.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v49, v49
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v26
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v51
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v11.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v26.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v49, v49
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v48
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v28.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v10.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v50, v50
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v25
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v27.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v32.l, v86.l, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v26.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v25.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v50, v50
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v13.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v52
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v53
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s1
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v51, v49
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v48
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v52, v50
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v25.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v9.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v12.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v27.l
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v26.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v29.l, v13.l, s3
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v49, v48
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v51
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v28.l, v12.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v50
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v11.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v48
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v10.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v49, v49
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v24
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v25.l
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v24.l, s3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v49
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v27.l, v11.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v26.l, v10.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v24.l, v8.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v23
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v49, v49
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v48
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v8.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v23.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v6
+; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v23.l, v7.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v25.l, v9.l, s1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v54.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.h, v31.l, v26.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v54.h, v28.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v26.l, v27.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v36
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v25
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v27.h
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v27, v54
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v28, v28
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v27.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v54.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.h, v36.l, v25.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v54.h, v27.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v25.l, v26.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v33
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v24
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v26.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v26, v54
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v27, v27
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v26.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v22
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v26, v24
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v7.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v22.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v10.l, v8.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v22.l, v6.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v54.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.h, v33.l, v24.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v54.h, v26.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v24.l, v25.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v23
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v25.h
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v25, v54
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v26, v26
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v25.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v22
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.l, v23.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v54.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v54.h, v25.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v23.l, v7.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v6
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v7.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v24, v24
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v7, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v7.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.l, v22.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v54.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v50.l, v54.h, v7.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v22.l, v6.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v21
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v54
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v22, v22
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v6.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v20
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.l, v21.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v54.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.l, v54.h, v6.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v21.l, v5.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v5.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v50
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v5.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v19
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.l, v20.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v54.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v54.h, v5.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v20.l, v4.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v39
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v4.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.l, v19.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v54.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v54.h, v4.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v19.l, v3.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v21
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v9.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v24, v23
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v6.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v21.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v7.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v11.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v8.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v24, v23
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v21.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v6.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v10.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v9.l, v8.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v24, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.l, v11.l, v7.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v21.l, v5.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v20
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v7.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v20.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v6.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v19
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v20.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s0, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v19.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v11
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v4.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v19.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v10.l, v6.l, s3
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v20
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v7.l, v5.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v10, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v11
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v19, v19
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v18.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v7, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v17
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v32
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v3.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v17
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.l, v18.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v54.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.l, v54.h, v3.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v18.l, v2.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v2.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v4, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v35
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v2.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v17.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v54.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.l, v54.h, v2.h, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v16
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v18.l, v2.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v17.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v16.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v7, v7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v5.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v8.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v17.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v16.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v18
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v6.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v17, v16
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v19, v18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v20
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v9.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v8.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v17
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v2.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v11
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v16
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v1.l
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s6, s7
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s5, s8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.l, v8.l, v0.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.l, v7.l, v1.l, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.l, v5.l, v2.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.l, v9.l, v3.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v6.l, v4.l, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v39 :: v_dual_mov_b32 v1, v38
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v37 :: v_dual_mov_b32 v3, v36
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v17.l, v1.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v3, v3
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v38
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v1.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v49
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v16.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v54.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.l, v54.h, v1.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v16.l, v0.h, s2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v52
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v54
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v53.l, v54.h, v0.h, s0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v53
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v34 :: v_dual_mov_b32 v5, v33
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v32 :: v_dual_mov_b32 v7, v31
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v30 :: v_dual_mov_b32 v9, v29
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, v28 :: v_dual_mov_b32 v11, v35
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v32bf16:
@@ -11114,578 +11486,665 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: scratch_load_b32 v55, off, s32
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v53, v15 :: v_dual_mov_b32 v48, v13
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v31, v10
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v33, v8
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v54.l, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v51, v14 :: v_dual_and_b32 v8, 0xffff0000, v53
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v34, v11 :: v_dual_mov_b32 v37, v12
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v36, v9
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v51
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v30
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v24
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v23
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v22
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v48
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v19
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v18
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v17
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v16
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v66.l, v54.l
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v55
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v53.h, v55.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v54.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v55.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX12-TRUE16-NEXT: scratch_load_b32 v31, off, s32
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v10
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v29
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v26
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v13
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v49, v49
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v36, v36
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v50, v50
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v67, v67
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.l, v10.h, v26.h, s9
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s43, v98, v98
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff0000, v12
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v30
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v69, v69
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v67.l, v26.h, v36.l, s10
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v98.l, v36.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v37, v37
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v28
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v50.l, v4.h, v20.h, s21
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v132.l, v67.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v98
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v34, v34
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v12.h, v28.h, s5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v36.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v132
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v38, v38
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v28
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v25
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s21, v98, v132
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v65, v65
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v65.l, v28.h, v34.l, s6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v21
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v52, v52
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v67.l, v67.l, v36.l, s21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v29
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v20
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v68, v68
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v98.l, v67.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s44, v99, v99
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v70, v70
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v13
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v81, v81
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v98
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v14
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v81.l, v20.h, v50.l, s22
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v35, v35
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v23
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s21, 0, v98
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v33, v33
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v22
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v13.h, v29.h, s3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v64, v64
+; GFX12-TRUE16-NEXT: s_and_b32 s5, s21, s5
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v8.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v51.h, v30.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v29
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v37
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v30.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.h, v67.l, v36.l, s5
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v64.l, v29.h, v33.l, s4
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v66, v66
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v71, v71
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.l, v2.h, v18.h, s25
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v51, v51
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v15
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v14
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v85, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.l, v9.h, v25.h, s11
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v14.h, v30.h, s1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v16
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v55, v55
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v68.l, v25.h, v37.l, s12
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v99.l, v37.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v37.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v55.l, v30.h, v32.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v96, v96
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v133.l, v68.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v99
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v29.l, s43
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v96.l, v34.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v130.l, v65.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v133, 16, v133
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v24
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v15
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v85, v85
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v86, v86
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s22, v99, v133
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v86.l, v32.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v128.l, v55.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s44
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v68.l, v68.l, v37.l, s22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v130
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v18
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v54, v54
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v87, v87
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v99.l, v68.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.l, v0.h, v16.h, s29
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v87.l, v33.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v129.l, v64.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v99
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v128
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v30
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.l, v5.h, v21.h, s19
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v118.l, v13.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s22, 0, v99
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s19, v96, v130
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v96.l, v29.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v82, v82
+; GFX12-TRUE16-NEXT: s_and_b32 s6, s22, s6
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v85.l, v16.h, v54.l, s40
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v8.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v48.h, v29.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v28
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v34
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v29.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.h, v68.l, v37.l, s6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v27
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v87
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v97, v97
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v118
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v48, v48
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v48.l, v6.h, v22.h, s17
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s17, v86, v128
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v129
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v65.l, v65.l, v34.l, s19
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v83, v83
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v71.l, v22.h, v48.l, s18
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v102.l, v48.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0x8000, v48.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v83.l, v18.h, v52.l, s26
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v116.l, v54.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v144.l, v71.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v102, 16, v102
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s18, v87, v128
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v87.l, v85.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v19
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v144, 16, v144
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s43, v118, v96
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v96.l, v65.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v8
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v114.l, v52.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s25, v102, v144
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v86.l, v83.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v116
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v87
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v64.l, v64.l, v33.l, s18
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v71.l, v71.l, v48.l, s25
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v80, v80
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v80.l, v21.h, v49.l, s20
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v53, v53
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v102.l, v71.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v30.l, s41
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v114
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s41, v116, v87
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v102, 16, v102
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v55.l, v55.l, v32.l, s17
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v87.l, v64.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s19, 0, v96
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.l, v8.h, v24.h, s13
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s25, 0, v102
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s29, v114, v86
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v86.l, v55.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v87
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v12
+; GFX12-TRUE16-NEXT: s_and_b32 s9, s25, s9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v69.l, v24.h, v38.l, s14
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v8.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v37.h, v28.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v27
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v31
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v28.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v8.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v34.h, v27.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v26
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v36
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v27.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v8.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v31.h, v26.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v25
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v33
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v26.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v8.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v36.h, v25.h, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v25.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v71.l, v48.l, s9
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s18, 0, v87
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s45, v100, v100
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v100.l, v38.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v39, v39
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v103.l, v49.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v112.l, v50.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s42
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v134.l, v69.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v11.h, v27.h, s7
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v145.l, v80.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v146.l, v81.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s17, 0, v86
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.l, v3.h, v19.h, s23
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v66.l, v27.h, v35.l, s8
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v97.l, v35.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v117.l, v14.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v100
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v103
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v131.l, v66.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v97
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v112
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v128.l, v30.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v134, 16, v134
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v131
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v145, 16, v145
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v146, 16, v146
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v84, 0xffff0000, v17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v53.l, v1.h, v17.h, s27
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s20, v97, v131
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v82.l, v19.h, v51.l, s24
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v117
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v128
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s23, v100, v134
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v66.l, v66.l, v35.l, s20
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s26, v103, v145
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s27, v112, v146
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v84, v84
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v113.l, v51.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v97.l, v66.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v147.l, v82.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s42, v117, v128
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v69.l, v69.l, v38.l, s23
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v80.l, v80.l, v49.l, s26
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v97
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v81.l, v81.l, v50.l, s27
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v32.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v84.l, v17.h, v53.l, s28
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v113
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s20, 0, v97
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v147, 16, v147
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s42
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v100.l, v69.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v103.l, v80.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v112.l, v81.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s46, v101, v101
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v28.l, s45
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v115.l, v53.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v129.l, v84.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s28, v113, v147
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v117.l, v30.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v100
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v103
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v112
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s17, s1
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v33.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s46
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v8.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v33.h, v24.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v7
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v24.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v32, v32
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v32.l, v54.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v8
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v55.l, v32.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.l, v7.h, v23.h, s15
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v38.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0x8000, v49.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0x8000, v50.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v115
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v129, 16, v129
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v82.l, v82.l, v51.l, s28
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v117
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s23, 0, v100
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s26, 0, v103
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s27, 0, v112
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s18, s2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v119.l, v12.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v130.l, v28.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v8.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v7.h, v23.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v23.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v35, v35
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v32
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v32.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v32
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v64.l, v33.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v70.l, v23.h, v39.l, s16
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0x8000, v14.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s40, v115, v129
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v113.l, v82.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s42, 0, v117
+; GFX12-TRUE16-NEXT: s_and_b32 s7, s23, s7
+; GFX12-TRUE16-NEXT: s_and_b32 s10, s26, s10
+; GFX12-TRUE16-NEXT: s_and_b32 s11, s27, s11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v50.h, v32.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v6.h, v22.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v21
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v35, v35
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v22.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v32
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v32.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v32
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.h, v69.l, v38.l, s7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.h, v80.l, v49.l, s10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.h, v81.l, v50.l, s11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v130
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v119
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v83.l, v83.l, v52.l, s29
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v84.l, v84.l, v53.l, s40
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v113
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v11
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0x8000, v51.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v114.l, v83.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v115.l, v84.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s28, 0, v113
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s43
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0x8000, v52.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v114
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v115
+; GFX12-TRUE16-NEXT: s_and_b32 s12, s28, s12
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0x8000, v53.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v34.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s29, 0, v114
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s40, 0, v115
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v35.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v101.l, v39.l
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s19, s3
+; GFX12-TRUE16-NEXT: s_and_b32 s13, s29, s13
+; GFX12-TRUE16-NEXT: s_and_b32 s14, s40, s14
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.h, v32.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v5.h, v21.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v20
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v35, v35
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v35.l, v54.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v21.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v32
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v32.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v32
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v65.l, v34.l, s3
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s20, s4
+; GFX12-TRUE16-NEXT: v_mov_b16_e64 v135.l, v70.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v32.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v4.h, v20.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.h, v66.l, v35.l, s4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v101
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v85.l, v85.l, v54.l, s41
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v39.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v135, 16, v135
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0x8000, v54.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v116.l, v85.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s24, v101, v135
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v116
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v70.l, v70.l, v39.l, s24
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.h, v20.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v38, v38
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v38.l, v54.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v35
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.h, v35.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v35
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v49, v49
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s41, 0, v116
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v101.l, v70.l
+; GFX12-TRUE16-NEXT: s_and_b32 s15, s41, s15
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v101
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.h, v35.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v3.h, v19.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.h, v19.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v49, v49
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v49.l, v54.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v38
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.h, v38.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v38
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v52, v52
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.h, v85.l, v54.l, s15
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v31
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v87, 0xffff0000, v31
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v86.l, v15.h, v31.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s24, 0, v101
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v96, v96
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v97.l, v15.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v87, v87
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v96.l, v86.l
+; GFX12-TRUE16-NEXT: s_and_b32 s8, s24, s8
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.l, v31.l, v15.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v97
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v87.l, v31.h, v86.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.h, v38.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v2.h, v18.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.h, v70.l, v39.l, s8
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v99.l, v31.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v98.l, v87.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.h, v18.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v52, v52
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v52.l, v54.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v49
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.h, v49.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v49
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.h, v49.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v1.h, v17.h, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v99
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v98
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.h, v17.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v64, v64
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v64.l, v54.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v52
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.h, v52.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v52
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.h, v52.h, v54.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v0.h, v16.h, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v53
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v64.h, v16.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v54.h
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v54, v64
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v64.h, v64.h, v54.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v55
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v64
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v66.h, v53.l, v55.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v97, v99
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v96, v98
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.l, v31.l, v15.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v15.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v87.l, v86.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v86.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v37.l, v31.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v36.l, v32.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v37
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v36
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v82.l, v51.l, s12
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v51.l, v29.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v37
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.h, v83.l, v52.l, s13
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v38
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.h, v84.l, v53.l, s14
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v53.h, v64.h, v54.h, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v55.l, v66.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v51
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v30
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v66.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v66, v54
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v64, v64
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v66.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v29
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.h, v51.l, v30.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v51.l, v54.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v31.l, v15.l, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s42, s16
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v54.h, v66.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v30.l, v51.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v48
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v51.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v55, v55
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v51, v54
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v30.l, v14.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v50, v49
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v27
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v51.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v30.l, v54.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v27.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v9
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v49, v49
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v26
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v51
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v51.l, v11.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v26.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v49, v49
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.h, v48.l, v29.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v28
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v48
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v48.l, v28.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v53.l, v10.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v50, v50
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v25
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v49.l, v27.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v32.l, v86.l, s1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v52.l, v26.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v25.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v50, v50
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v13.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v52
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v53
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s1
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v51, v49
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v48
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v52, v50
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v48.l, v25.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v49.l, v9.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v12.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v48
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v49
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v50.l, v27.l
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v51.l, v26.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v54.h, v51.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v29.l, v30.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v37
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v30.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v48, v48
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v30, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v30.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v29.l, v54.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.h, v37.l, v28.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v29.l, v13.l, s3
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v49, v48
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v51
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v28.l, v12.l, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v50
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v11.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v48
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v10.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v49, v49
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v24
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v49.l, v25.l
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX12-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v24.l, s3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v49
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v54.h, v30.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v28.l, v29.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v34
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v27
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v29.h
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v29, v54
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v30, v30
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v29.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v54.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.h, v34.l, v27.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v27.l, v11.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v26.l, v10.l, s1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v24.l, v8.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v23
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v49, v49
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v48
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v9.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v10.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v8.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v23.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v6
+; GFX12-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v23.l, v7.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v54.h, v29.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v27.l, v28.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v31
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v26
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v28.h
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v28, v54
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v29, v29
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v28.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v25.l, v9.l, s1
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v54.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.h, v31.l, v26.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v54.h, v28.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v26.l, v27.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v36
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v25
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v27.h
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v27, v54
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v28, v28
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v27.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v54.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v22
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v26, v24
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v11.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v7.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.h, v36.l, v25.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v54.h, v27.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v25.l, v26.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v33
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v24
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v26.h
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v26, v54
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v27, v27
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v22.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v10.l, v8.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v26.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v22.l, v6.l, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v54.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.h, v33.l, v24.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v54.h, v26.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v24.l, v25.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v23
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v25.h
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v25, v54
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v26, v26
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v25.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v22
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v21
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v9.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v24, v23
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v10.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v6.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.l, v23.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v54.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v21.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v7.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v11.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v8.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v24, v23
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v21.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v5.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v6.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v7.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v10.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v54.h, v25.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v23.l, v7.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v6
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v7.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v24, v24
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v7, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v7.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.l, v22.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v54.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v9.l, v8.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v24, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v22
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v50.l, v54.h, v7.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v22.l, v6.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v21
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v54
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v22, v22
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v6.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v20
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.l, v11.l, v7.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v21.l, v5.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v20
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v7.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.l, v21.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v54.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.l, v54.h, v6.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v21.l, v5.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v5.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v50
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v54
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v20.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v6.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v19
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v5.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v19
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v20.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s0, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v19.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v11
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v5.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v4.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.l, v20.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v54.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v19.l, v3.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v54.h, v5.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v20.l, v4.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v39
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v4.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v18
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.l, v19.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v54.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v10.l, v6.l, s3
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v20
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v9.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v54.h, v4.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v19.l, v3.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.h
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v7.l, v5.l, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v10, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v11
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v19, v19
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v18
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v18.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v7, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v17
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v32
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v3.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v17
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.l, v18.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v54.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.l, v54.h, v3.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v18.l, v2.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v2.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v4, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v35
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v2.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v16
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v16
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v18.l, v2.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v17.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v16.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v7, v7
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v5.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v17.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v54.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v8.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v17.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v16.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v18
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v7.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v8.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v0.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v6.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v10
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v3.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v17, v16
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v19, v18
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v20
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v9.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v5.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v7.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v8.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v17
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v2.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v11
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v16
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v1.l
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s6, s7
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s5, s8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.l, v54.h, v2.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v17.l, v1.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v3, v3
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v38
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v49
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.l, v8.l, v0.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.l, v7.l, v1.l, s4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.l, v5.l, v2.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.l, v9.l, v3.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v6.l, v4.l, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v39 :: v_dual_mov_b32 v1, v38
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, v37 :: v_dual_mov_b32 v3, v36
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v16.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v54.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.l, v54.h, v1.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v16.l, v0.h, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v52
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v54
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.h, v54.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v54
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v53.l, v54.h, v0.h, s0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v53
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v34 :: v_dual_mov_b32 v5, v33
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v6, v32 :: v_dual_mov_b32 v7, v31
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v8, v30 :: v_dual_mov_b32 v9, v29
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v10, v28 :: v_dual_mov_b32 v11, v35
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v32bf16:
@@ -12447,25 +12906,31 @@ define bfloat @v_minimumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
; GFX11-TRUE16-LABEL: v_minimumnum_bf16_no_ieee:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v0.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v1.h, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.h, v1.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v1.h, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_bf16_no_ieee:
@@ -12499,28 +12964,34 @@ define bfloat @v_minimumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, v0.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.h
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v1.h, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_bf16_no_ieee:
@@ -12734,38 +13205,52 @@ define <2 x bfloat> @v_minimumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v1.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.h, v2.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v2.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.h, v2.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.l, v4.h, s2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.h, v4.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v4.h, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v2.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v2.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v2bf16_no_ieee:
@@ -12819,44 +13304,56 @@ define <2 x bfloat> @v_minimumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v0.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v1.h, v2.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v2.h
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v3.h, v2.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.h, v2.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.l, v4.h, s2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.h
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v6
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v2.h, v4.h, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v2.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.h, v4.h, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.l, v2.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v2bf16_no_ieee:
@@ -13160,54 +13657,67 @@ define <3 x bfloat> @v_minimumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v8, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.h, v2.h, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v2.h, v4.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v4.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v5.h, v4.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.l, v1.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v1.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.h, v1.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.l, v0.h, s2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v4.h, v0.h, s0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v4.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v9
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v10, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s3, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s4, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v3bf16_no_ieee:
@@ -13276,62 +13786,72 @@ define <3 x bfloat> @v_minimumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v8, v8
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v0.h, v2.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v2.h, v4.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.h
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v4.l, s1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v8
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v9
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v10, v11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v4.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v5.h, v4.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.l, v1.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.h
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v1, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s3, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.h, v1.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v2.l, v0.h, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s4, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v4.h, v0.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v3bf16_no_ieee:
@@ -13731,70 +14251,77 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v1.h, v3.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.h, v6.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v6.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v6.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v0.h, v2.h, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v2.h, v6.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v6.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v10, v10
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v1.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.h, v6.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v3.l, v7.h, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v7.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v7, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v7.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v6.h, v7.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.l, v0.h, s2
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.h, v0.h, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v1 :: v_dual_mov_b32 v1, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v4.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v4.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v1.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v10, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v9
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v12, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v4.l, s5
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v10
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v6.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v4bf16_no_ieee:
@@ -13887,81 +14414,86 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v1.h, v3.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.h, v6.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v6.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v4.h, v6.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v0.h, v2.h, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v2.h, v6.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.h
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v5
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v5.h, v6.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v10, v10
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v4.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v4.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v6.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v9
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v10, v8
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v9
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v12, v13
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v1.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.h, v6.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v3.l, v7.h, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v7.h
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v7.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v4.l, s5
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v10
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v6.h, v7.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v2.l, v0.h, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v0, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v6.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.h
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v6.l, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.h, v0.h, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v1 :: v_dual_mov_b32 v1, v4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v4bf16_no_ieee:
diff --git a/llvm/test/CodeGen/AMDGPU/minmax3-tree-reduction.ll b/llvm/test/CodeGen/AMDGPU/minmax3-tree-reduction.ll
index f18864fe3d26a..88c38ccbb1508 100644
--- a/llvm/test/CodeGen/AMDGPU/minmax3-tree-reduction.ll
+++ b/llvm/test/CodeGen/AMDGPU/minmax3-tree-reduction.ll
@@ -518,49 +518,22 @@ define bfloat @v_no_max3_maxnum_tree4_bf16(bfloat %a, bfloat %b, bfloat %c, bflo
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250-FAKE16-LABEL: v_no_max3_maxnum_tree4_bf16:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX1250-FAKE16-NEXT: v_dual_lshlrev_b32 v2, 16, v2 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_dual_max_num_f32 v2, v2, v3 :: v_dual_max_num_f32 v0, v0, v1
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v1, v2, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-FAKE16-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_max_num_f32_e32 v0, v0, v1
-; GFX1250-FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-REAL16-LABEL: v_no_max3_maxnum_tree4_bf16:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v4.l, 0
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v4.h, v1.l
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v1.l, v4.l
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v2.l, v4.l
-; GFX1250-REAL16-NEXT: v_max_num_f32_e32 v1, v1, v4
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v4.h, v3.l
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-REAL16-NEXT: v_max_num_f32_e32 v0, v0, v4
-; GFX1250-REAL16-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-REAL16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v4.h, v0.l
-; GFX1250-REAL16-NEXT: v_max_num_f32_e32 v0, v2, v4
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-REAL16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_no_max3_maxnum_tree4_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX1250-NEXT: v_dual_lshlrev_b32 v2, 16, v2 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_dual_max_num_f32 v2, v2, v3 :: v_dual_max_num_f32 v0, v0, v1
+; GFX1250-NEXT: v_cvt_pk_bf16_f32 v1, v2, s0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: v_dual_lshlrev_b32 v1, 16, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_max_num_f32_e32 v0, v0, v1
+; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%max.ab = call bfloat @llvm.maxnum.bf16(bfloat %a, bfloat %b)
%max.cd = call bfloat @llvm.maxnum.bf16(bfloat %c, bfloat %d)
%result = call bfloat @llvm.maxnum.bf16(bfloat %max.ab, bfloat %max.cd)
More information about the llvm-commits
mailing list