[llvm] [AMDGPU][True16][Codegen] remove another build_vector pattern from true16 (PR #149861)
Brox Chen via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 25 14:34:09 PDT 2025
https://github.com/broxigarchen updated https://github.com/llvm/llvm-project/pull/149861
>From 8a22470a949ece84041de7ac858899c900b9ab48 Mon Sep 17 00:00:00 2001
From: guochen2 <guochen2 at amd.com>
Date: Mon, 21 Jul 2025 13:50:17 -0400
Subject: [PATCH] remove build vector
---
llvm/lib/Target/AMDGPU/SIInstructions.td | 26 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll | 7569 +++++++++--------
.../CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll | 745 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll | 1641 ++--
.../CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll | 131 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll | 61 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll | 3565 ++++----
.../CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll | 405 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll | 421 +-
.../atomic_optimizations_global_pointer.ll | 5181 +++++++----
llvm/test/CodeGen/AMDGPU/bf16.ll | 4449 +++++-----
.../CodeGen/AMDGPU/bitcast_vector_bigint.ll | 5 -
.../buffer-fat-pointer-atomicrmw-fadd.ll | 63 +-
.../buffer-fat-pointer-atomicrmw-fmax.ll | 45 +-
.../buffer-fat-pointer-atomicrmw-fmin.ll | 45 +-
.../build-vector-packed-partial-undef.ll | 27 +-
.../CodeGen/AMDGPU/build_vector.gfx11plus.ll | 145 +
llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll | 18 +-
.../CodeGen/AMDGPU/dagcombine-fmul-sel.ll | 80 +-
.../AMDGPU/divergence-driven-buildvector.ll | 18 +-
llvm/test/CodeGen/AMDGPU/fabs.bf16.ll | 74 +-
llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll | 208 +-
.../CodeGen/AMDGPU/flat-atomicrmw-fadd.ll | 120 +-
.../CodeGen/AMDGPU/flat-atomicrmw-fmax.ll | 140 +-
.../CodeGen/AMDGPU/flat-atomicrmw-fmin.ll | 140 +-
.../CodeGen/AMDGPU/flat-atomicrmw-fsub.ll | 140 +-
.../CodeGen/AMDGPU/global-atomicrmw-fadd.ll | 140 +-
.../CodeGen/AMDGPU/global-atomicrmw-fmax.ll | 140 +-
.../CodeGen/AMDGPU/global-atomicrmw-fmin.ll | 140 +-
.../CodeGen/AMDGPU/global-atomicrmw-fsub.ll | 140 +-
llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll | 418 +-
.../CodeGen/AMDGPU/insert_vector_elt.v2i16.ll | 366 +-
llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll | 84 +-
.../CodeGen/AMDGPU/local-atomicrmw-fadd.ll | 36 +-
.../CodeGen/AMDGPU/local-atomicrmw-fmax.ll | 62 +-
.../CodeGen/AMDGPU/local-atomicrmw-fmin.ll | 62 +-
.../CodeGen/AMDGPU/local-atomicrmw-fsub.ll | 62 +-
.../test/CodeGen/AMDGPU/vector_rebroadcast.ll | 74 +-
.../CodeGen/AMDGPU/vector_shuffle.packed.ll | 158 +-
39 files changed, 14499 insertions(+), 12845 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/build_vector.gfx11plus.ll
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index e8b4501226732..06e40236f65b2 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -2174,7 +2174,6 @@ def : GCNPat <
}
foreach fp16vt = [f16, bf16] in {
-
def : GCNPat <
(fcopysign fp16vt:$src0, fp16vt:$src1),
(V_BFI_B32_e64 (S_MOV_B32 (i32 0x00007fff)), $src0, $src1)
@@ -3637,13 +3636,24 @@ def : GCNPat <
>;
foreach p = [NotHasTrue16BitInsts, UseFakeTrue16Insts] in
-let True16Predicate = p in
+let True16Predicate = p in {
// Take the lower 16 bits from each VGPR_32 and concat them
def : GCNPat <
(vecTy (DivergentBinFrag<build_vector> (Ty VGPR_32:$a), (Ty VGPR_32:$b))),
(V_PERM_B32_e64 VGPR_32:$b, VGPR_32:$a, (S_MOV_B32 (i32 0x05040100)))
>;
+// Take the lower 16 bits from V[0] and the upper 16 bits from V[1]
+// Special case, can use V_BFI (0xffff literal likely more reusable than 0x70601000)
+def : GCNPat <
+ (vecTy (DivergentBinFrag<build_vector> (Ty VGPR_32:$a),
+ (Ty !if(!eq(Ty, i16),
+ (Ty (trunc (srl VGPR_32:$b, (i32 16)))),
+ (Ty (bitconvert (i16 (trunc (srl VGPR_32:$b, (i32 16)))))))))),
+ (V_BFI_B32_e64 (S_MOV_B32 (i32 0x0000ffff)), VGPR_32:$a, VGPR_32:$b)
+>;
+}
+
let True16Predicate = UseRealTrue16Insts in {
def : GCNPat <
(vecTy (DivergentBinFrag<build_vector> (Ty VGPR_16:$a), (Ty VGPR_16:$b))),
@@ -3669,18 +3679,6 @@ def : GCNPat <
(V_AND_B32_e64 (S_MOV_B32 (i32 0xffff0000)), VGPR_32:$b)
>;
-
-// Take the lower 16 bits from V[0] and the upper 16 bits from V[1]
-// Special case, can use V_BFI (0xffff literal likely more reusable than 0x70601000)
-def : GCNPat <
- (vecTy (DivergentBinFrag<build_vector> (Ty VGPR_32:$a),
- (Ty !if(!eq(Ty, i16),
- (Ty (trunc (srl VGPR_32:$b, (i32 16)))),
- (Ty (bitconvert (i16 (trunc (srl VGPR_32:$b, (i32 16)))))))))),
- (V_BFI_B32_e64 (S_MOV_B32 (i32 0x0000ffff)), VGPR_32:$a, VGPR_32:$b)
->;
-
-
// Take the upper 16 bits from V[0] and the lower 16 bits from V[1]
// Special case, can use V_ALIGNBIT (always uses encoded literal)
let True16Predicate = NotHasTrue16BitInsts in {
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
index d03d6a8940b2f..46b82d3a3d651 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
@@ -25961,22 +25961,64 @@ define <32 x i32> @bitcast_v64bf16_to_v32i32(<64 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v64bf16_to_v32i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_clause 0xf
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v40, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v42, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v43, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v44, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v45, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v46, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v47, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v56, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v57, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v58, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v59, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v60, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v61, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v62, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v63, s32 offset:8
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: scratch_load_b32 v32, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v32
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v32
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB18_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v63, v31 :: v_dual_mov_b32 v62, v30
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v61, v29 :: v_dual_mov_b32 v60, v28
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v59, v27 :: v_dual_mov_b32 v58, v26
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v57, v25 :: v_dual_mov_b32 v56, v24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v55, v23 :: v_dual_mov_b32 v54, v22
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v53, v21 :: v_dual_mov_b32 v52, v20
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v51, v19 :: v_dual_mov_b32 v50, v18
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v49, v17 :: v_dual_mov_b32 v48, v16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v47, v15 :: v_dual_mov_b32 v46, v14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v45, v13 :: v_dual_mov_b32 v44, v12
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v43, v11 :: v_dual_mov_b32 v42, v10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v41, v9 :: v_dual_mov_b32 v40, v8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, v7 :: v_dual_mov_b32 v38, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v37, v5 :: v_dual_mov_b32 v36, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, v3 :: v_dual_mov_b32 v34, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, v1 :: v_dual_mov_b32 v32, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: .LBB18_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB18_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v33, 0x40c00000, v33 :: v_dual_lshlrev_b32 v32, 16, v15
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v15
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v33, 0x40c00000, v33 :: v_dual_add_f32 v32, 0x40c00000, v32
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v33, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v32, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v32
@@ -25984,539 +26026,551 @@ define <32 x i32> @bitcast_v64bf16_to_v32i32(<64 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v32, 0x7fff
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v32, v35, v38 :: v_dual_and_b32 v15, 0xffff0000, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_dual_add_f32 v15, 0x40c00000, v15 :: v_dual_lshlrev_b32 v14, 16, v14
; GFX11-TRUE16-NEXT: v_add3_u32 v35, v39, v33, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v32.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v15, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v15
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v36, v36, v15, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_cndmask_b32 v15, v36, v37
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_cndmask_b32 v47, v36, v37
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_bfe_u32 v48, v14, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v49, 0x400000, v14
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v33
-; GFX11-TRUE16-NEXT: v_bfi_b32 v15, 0xffff, v32, v15
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v47.l, v32.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v32, 0x400000, v33
; GFX11-TRUE16-NEXT: v_add3_u32 v38, v48, v14, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v38, v49, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v14.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v33, v35, v36 :: v_dual_add_f32 v34, 0x40c00000, v34
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v14, 0xffff, v14, v33
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v34, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v38, v13, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v34, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v36, v38, v13, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v13
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v13, v36, v37 :: v_dual_and_b32 v38, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v13.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v35, v39, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v13, 0xffff, v13, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v15, 0x40c00000, v34 :: v_dual_cndmask_b32 v46, v35, v32
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v15, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v46.l, v14.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v12
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v36
+; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v13, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v33, v34, v15, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v34, 0x400000, v13
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v35, 0x400000, v15
+; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v13, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v32, v34, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_cndmask_b32 v45, v33, v35
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v45.l, v13.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v12, 0x40c00000, v12 :: v_dual_lshlrev_b32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v14, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v34, 0x400000, v14
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_add_f32 v11, 0x40c00000, v11
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v12, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v35, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v12
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v12, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v32, v35, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v11, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v14, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v12, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v15, v33, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v13, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v44, v32, v34, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v32, 0x400000, v11
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v11.h
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v33, v33, v38 :: v_dual_and_b32 v38, 0xffff0000, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v44.l, v12.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v13, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v11, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v11, v14, v32 :: v_dual_and_b32 v12, 0xffff0000, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v11, 0xffff, v11, v33
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v10, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v12, 0x40c00000, v12 :: v_dual_cndmask_b32 v43, v15, v33
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v43.l, v11.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v10, 0x40c00000, v10 :: v_dual_lshlrev_b32 v9, 16, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v32, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v10
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v10, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v12.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v10.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: v_bfi_b32 v12, 0xffff, v12, v32
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v12, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v10, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v13, v15, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v42, v14, v32, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v9
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v10, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v42.l, v10.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v9, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v12, v14 :: v_dual_and_b32 v10, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v9, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_cndmask_b32 v9, v32, v36
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_add_f32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v10, 0x40c00000, v10 :: v_dual_cndmask_b32 v41, v13, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v41.l, v9.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_lshlrev_b32 v7, 16, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v8
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v9.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v8, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v11, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v40, v12, v14, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v9, 0xffff, v9, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v40.l, v8.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v10, v12 :: v_dual_and_b32 v8, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_bfi_b32 v8, 0xffff, v8, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_cndmask_b32 v39, v11, v13
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v7.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v7.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v7, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_add_f32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v32, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v8, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v38, v10, v12, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v6, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v6.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v7, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v8, v10 :: v_dual_and_b32 v6, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v37, v9, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v5, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v36, v8, v10, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v4.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v6, v8 :: v_dual_and_b32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v34, 0x40c00000, v34
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_cndmask_b32 v3, v32, v36
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_add_f32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_cndmask_b32 v35, v7, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v3.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v2.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v4, v6 :: v_dual_and_b32 v2, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v35, 0x40c00000, v38 :: v_dual_add_f32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v34
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v31
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_add_f32 v31, 0x40c00000, v31
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v32, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v31, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v30
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v31, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v31
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v31, v31
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v31, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v30, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v31.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v30, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v30
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v29
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; GFX11-TRUE16-NEXT: v_bfi_b32 v31, 0xffff, v31, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v30, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v30.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v29, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v30, 0xffff, v30, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v28
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v29, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v29
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v34, 0x40c00000, v34
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v29, v32, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v28, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v28, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v28
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_add_f32 v27, 0x40c00000, v27
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v29.h
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_cndmask_b32 v33, v5, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v28, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v27, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v29, 0xffff, v29, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v28.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v27, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v27
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; GFX11-TRUE16-NEXT: v_bfi_b32 v28, 0xffff, v28, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v27, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v27.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v26, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v27, 0xffff, v27, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v26, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v26
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_add_f32 v25, 0x40c00000, v25
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v26, v32, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v1.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v31
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v5, 16, v31
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v4, v7, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v25, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v24
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v25, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v25
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v26.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v25, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v24, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v26, 0xffff, v26, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v25.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v24, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v24
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX11-TRUE16-NEXT: v_bfi_b32 v25, 0xffff, v25, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v24, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v24.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v23, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v24, 0xffff, v24, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v23, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v23
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v34, 0x40c00000, v34
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v20
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v23, v32, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v22, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v21
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v22, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v22
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_add_f32 v21, 0x40c00000, v21
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v23.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v22, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v21, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v23, 0xffff, v23, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v22.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v21, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v21
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v36
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v22, 0xffff, v22, v33
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v20, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v21, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v39
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v20
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v20, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v21.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v35, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v18
-; GFX11-TRUE16-NEXT: v_bfi_b32 v21, 0xffff, v21, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v33, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_add_f32 v19, 0x40c00000, v19
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v32, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v19, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v19
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v36, v36, v19, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v35, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v20.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v19, v36, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v39, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v18, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v19.h
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v33, v33, v38 :: v_dual_and_b32 v38, 0xffff0000, v17
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v18, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfi_b32 v20, 0xffff, v20, v32
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v37, 0x40c00000, v38 :: v_dual_cndmask_b32 v34, v34, v36
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v16
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v48, 0x400000, v37
-; GFX11-TRUE16-NEXT: v_bfi_b32 v19, 0xffff, v19, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v35, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_bfe_u32 v38, v17, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v49, 0x400000, v17
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v37, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v38, v38, v17, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v51, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v37, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v39, v39, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v38, v49, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_bfe_u32 v50, v16, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v49, 0x400000, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v18.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v17.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v36, v39, v51, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX11-TRUE16-NEXT: v_add3_u32 v38, v50, v16, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfi_b32 v18, 0xffff, v18, v34
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v36.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v35, v35, v48, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT: v_bfi_b32 v17, 0xffff, v17, v35
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v38, v49, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v0.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v63, v3, v7 :: v_dual_and_b32 v0, 0xffff0000, v30
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v29
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v30
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v16, 0xffff, v36, v16
-; GFX11-TRUE16-NEXT: .LBB18_2: ; %end
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v62, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v62.l, v1.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v28
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v63.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v29
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v61, v3, v7 :: v_dual_lshlrev_b32 v4, 16, v28
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v27
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v61.l, v0.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v27
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v60, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v59, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v59.l, v1.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v25
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v60.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v26
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v25
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v4, 16, v26
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v58, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v4, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v24
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v58.l, v0.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v24
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v57, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v57.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v23
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v23
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_add_f32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v56, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v56.l, v1.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v22
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v22
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v55, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v21
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v0.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v21
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v5, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v54, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v2.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v20
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v20
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v5, 16, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v53, v2, v7, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v1.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v19
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v6, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v52, v0, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v4, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v3, v4 :: v_dual_lshlrev_b32 v6, 16, v18
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v6 :: v_dual_cndmask_b32 v51, v0, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v3.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v17
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v18
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v8, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v50, v0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v5, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v16
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v0, v1 :: v_dual_add_f32 v1, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v3, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v0.h
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v9, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v49, v2, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v0.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v48, v1, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v5.h
+; GFX11-TRUE16-NEXT: .LBB18_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v40 :: v_dual_mov_b32 v9, v41
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, v42 :: v_dual_mov_b32 v11, v43
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v44 :: v_dual_mov_b32 v13, v45
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, v46 :: v_dual_mov_b32 v15, v47
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v24, v56 :: v_dual_mov_b32 v25, v57
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, v58 :: v_dual_mov_b32 v27, v59
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v28, v60 :: v_dual_mov_b32 v29, v61
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v30, v62 :: v_dual_mov_b32 v31, v63
+; GFX11-TRUE16-NEXT: s_clause 0xf
+; GFX11-TRUE16-NEXT: scratch_load_b32 v63, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_b32 v62, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_b32 v61, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_b32 v60, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_b32 v59, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_b32 v58, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_b32 v57, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_b32 v56, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_b32 v47, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_b32 v46, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_b32 v45, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_b32 v44, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_b32 v43, off, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s32 offset:68
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v32 :: v_dual_mov_b32 v1, v33
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v34 :: v_dual_mov_b32 v3, v35
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v36 :: v_dual_mov_b32 v5, v37
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v38 :: v_dual_mov_b32 v7, v39
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, v48 :: v_dual_mov_b32 v17, v49
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, v50 :: v_dual_mov_b32 v19, v51
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v20, v52 :: v_dual_mov_b32 v21, v53
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v22, v54 :: v_dual_mov_b32 v23, v55
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -62676,22 +62730,64 @@ define <32 x float> @bitcast_v64bf16_to_v32f32(<64 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v64bf16_to_v32f32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_clause 0xf
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v40, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v42, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v43, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v44, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v45, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v46, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v47, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v56, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v57, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v58, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v59, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v60, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v61, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v62, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v63, s32 offset:8
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: scratch_load_b32 v32, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v32
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v32
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB42_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v63, v31 :: v_dual_mov_b32 v62, v30
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v61, v29 :: v_dual_mov_b32 v60, v28
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v59, v27 :: v_dual_mov_b32 v58, v26
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v57, v25 :: v_dual_mov_b32 v56, v24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v55, v23 :: v_dual_mov_b32 v54, v22
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v53, v21 :: v_dual_mov_b32 v52, v20
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v51, v19 :: v_dual_mov_b32 v50, v18
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v49, v17 :: v_dual_mov_b32 v48, v16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v47, v15 :: v_dual_mov_b32 v46, v14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v45, v13 :: v_dual_mov_b32 v44, v12
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v43, v11 :: v_dual_mov_b32 v42, v10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v41, v9 :: v_dual_mov_b32 v40, v8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, v7 :: v_dual_mov_b32 v38, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v37, v5 :: v_dual_mov_b32 v36, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, v3 :: v_dual_mov_b32 v34, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, v1 :: v_dual_mov_b32 v32, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: .LBB42_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB42_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v33, 0x40c00000, v33 :: v_dual_lshlrev_b32 v32, 16, v15
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v15
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v33, 0x40c00000, v33 :: v_dual_add_f32 v32, 0x40c00000, v32
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v33, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v32, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v32
@@ -62699,539 +62795,551 @@ define <32 x float> @bitcast_v64bf16_to_v32f32(<64 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v32, 0x7fff
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v32, v35, v38 :: v_dual_and_b32 v15, 0xffff0000, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_dual_add_f32 v15, 0x40c00000, v15 :: v_dual_lshlrev_b32 v14, 16, v14
; GFX11-TRUE16-NEXT: v_add3_u32 v35, v39, v33, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v32.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v15, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v15
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v36, v36, v15, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_cndmask_b32 v15, v36, v37
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_cndmask_b32 v47, v36, v37
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_bfe_u32 v48, v14, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v49, 0x400000, v14
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v33
-; GFX11-TRUE16-NEXT: v_bfi_b32 v15, 0xffff, v32, v15
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v47.l, v32.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v32, 0x400000, v33
; GFX11-TRUE16-NEXT: v_add3_u32 v38, v48, v14, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v38, v49, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v14.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v33, v35, v36 :: v_dual_add_f32 v34, 0x40c00000, v34
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v14, 0xffff, v14, v33
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v34, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v38, v13, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v34, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v36, v38, v13, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v13
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v13, v36, v37 :: v_dual_and_b32 v38, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v13.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v35, v39, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v13, 0xffff, v13, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v15, 0x40c00000, v34 :: v_dual_cndmask_b32 v46, v35, v32
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v15, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v46.l, v14.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v12
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v36
+; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v13, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v33, v34, v15, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v34, 0x400000, v13
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v35, 0x400000, v15
+; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v13, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v32, v34, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_cndmask_b32 v45, v33, v35
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v45.l, v13.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v12, 0x40c00000, v12 :: v_dual_lshlrev_b32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v14, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v34, 0x400000, v14
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_add_f32 v11, 0x40c00000, v11
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v12, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v35, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v12
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v12, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v32, v35, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v11, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v14, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v12, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v15, v33, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v13, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v44, v32, v34, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v32, 0x400000, v11
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v11.h
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v33, v33, v38 :: v_dual_and_b32 v38, 0xffff0000, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v44.l, v12.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v13, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v11, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v11, v14, v32 :: v_dual_and_b32 v12, 0xffff0000, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v11, 0xffff, v11, v33
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v10, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v12, 0x40c00000, v12 :: v_dual_cndmask_b32 v43, v15, v33
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v43.l, v11.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v10, 0x40c00000, v10 :: v_dual_lshlrev_b32 v9, 16, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v32, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v10
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v10, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v12.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v10.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: v_bfi_b32 v12, 0xffff, v12, v32
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v12, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v10, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v13, v15, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v42, v14, v32, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v9
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v10, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v42.l, v10.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v9, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v12, v14 :: v_dual_and_b32 v10, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v9, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_cndmask_b32 v9, v32, v36
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_add_f32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v10, 0x40c00000, v10 :: v_dual_cndmask_b32 v41, v13, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v41.l, v9.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_lshlrev_b32 v7, 16, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v8
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v9.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v8, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v11, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v40, v12, v14, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v9, 0xffff, v9, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v40.l, v8.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v10, v12 :: v_dual_and_b32 v8, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_bfi_b32 v8, 0xffff, v8, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_cndmask_b32 v39, v11, v13
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v7.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v7.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v7, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_add_f32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v32, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v8, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v38, v10, v12, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v6, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v6.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v7, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v8, v10 :: v_dual_and_b32 v6, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v37, v9, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v5, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v36, v8, v10, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v4.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v6, v8 :: v_dual_and_b32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v34, 0x40c00000, v34
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_cndmask_b32 v3, v32, v36
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_add_f32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_cndmask_b32 v35, v7, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v3.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v2.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v4, v6 :: v_dual_and_b32 v2, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v35, 0x40c00000, v38 :: v_dual_add_f32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v34
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v31
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_add_f32 v31, 0x40c00000, v31
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v32, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v31, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v30
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v31, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v31
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v31, v31
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v31, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v30, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v31.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v30, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v30
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v29
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; GFX11-TRUE16-NEXT: v_bfi_b32 v31, 0xffff, v31, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v30, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v30.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v29, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v30, 0xffff, v30, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v28
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v29, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v29
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v34, 0x40c00000, v34
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v29, v32, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v28, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v28, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v28
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_add_f32 v27, 0x40c00000, v27
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v29.h
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_cndmask_b32 v33, v5, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v28, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v27, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v29, 0xffff, v29, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v28.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v27, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v27
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; GFX11-TRUE16-NEXT: v_bfi_b32 v28, 0xffff, v28, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v27, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v27.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v26, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v27, 0xffff, v27, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v26, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v26
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_add_f32 v25, 0x40c00000, v25
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v26, v32, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v1.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v31
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v5, 16, v31
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v4, v7, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v25, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v24
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v25, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v25
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v26.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v25, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v24, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v26, 0xffff, v26, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v25.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v24, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v24
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX11-TRUE16-NEXT: v_bfi_b32 v25, 0xffff, v25, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v24, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v24.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v23, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v24, 0xffff, v24, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v23, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v23
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v34, 0x40c00000, v34
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v20
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v23, v32, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v22, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v21
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v22, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v22
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_add_f32 v21, 0x40c00000, v21
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v23.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v22, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v21, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v23, 0xffff, v23, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v22.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v21, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v21
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v36
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v22, 0xffff, v22, v33
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v20, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v21, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v39
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v20
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v20, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v21.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v35, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v18
-; GFX11-TRUE16-NEXT: v_bfi_b32 v21, 0xffff, v21, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v33, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_add_f32 v19, 0x40c00000, v19
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v32, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v19, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v19
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v36, v36, v19, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v35, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v20.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v19, v36, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v39, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v18, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v19.h
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v33, v33, v38 :: v_dual_and_b32 v38, 0xffff0000, v17
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v18, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfi_b32 v20, 0xffff, v20, v32
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v37, 0x40c00000, v38 :: v_dual_cndmask_b32 v34, v34, v36
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v16
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v48, 0x400000, v37
-; GFX11-TRUE16-NEXT: v_bfi_b32 v19, 0xffff, v19, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v35, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_bfe_u32 v38, v17, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v49, 0x400000, v17
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v37, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v38, v38, v17, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v51, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v37, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v39, v39, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v38, v49, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_bfe_u32 v50, v16, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v49, 0x400000, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v18.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v17.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v36, v39, v51, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX11-TRUE16-NEXT: v_add3_u32 v38, v50, v16, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfi_b32 v18, 0xffff, v18, v34
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v36.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v35, v35, v48, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT: v_bfi_b32 v17, 0xffff, v17, v35
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v38, v49, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v0.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v63, v3, v7 :: v_dual_and_b32 v0, 0xffff0000, v30
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v29
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v30
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v16, 0xffff, v36, v16
-; GFX11-TRUE16-NEXT: .LBB42_2: ; %end
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v62, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v62.l, v1.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v28
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v63.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v29
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v61, v3, v7 :: v_dual_lshlrev_b32 v4, 16, v28
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v27
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v61.l, v0.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v27
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v60, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v59, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v59.l, v1.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v25
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v60.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v26
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v25
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v4, 16, v26
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v58, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v4, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v24
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v58.l, v0.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v24
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v57, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v57.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v23
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v23
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_add_f32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v56, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v56.l, v1.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v22
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v22
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v55, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v21
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v0.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v21
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v5, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v54, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v2.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v20
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v20
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v5, 16, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v53, v2, v7, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v1.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v19
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v6, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v52, v0, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v4, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v3, v4 :: v_dual_lshlrev_b32 v6, 16, v18
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v6 :: v_dual_cndmask_b32 v51, v0, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v3.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v17
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v18
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v8, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v50, v0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v5, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v16
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v0, v1 :: v_dual_add_f32 v1, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v3, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v0.h
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v9, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v49, v2, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v0.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v48, v1, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v5.h
+; GFX11-TRUE16-NEXT: .LBB42_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v40 :: v_dual_mov_b32 v9, v41
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, v42 :: v_dual_mov_b32 v11, v43
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v44 :: v_dual_mov_b32 v13, v45
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, v46 :: v_dual_mov_b32 v15, v47
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v24, v56 :: v_dual_mov_b32 v25, v57
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, v58 :: v_dual_mov_b32 v27, v59
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v28, v60 :: v_dual_mov_b32 v29, v61
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v30, v62 :: v_dual_mov_b32 v31, v63
+; GFX11-TRUE16-NEXT: s_clause 0xf
+; GFX11-TRUE16-NEXT: scratch_load_b32 v63, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_b32 v62, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_b32 v61, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_b32 v60, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_b32 v59, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_b32 v58, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_b32 v57, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_b32 v56, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_b32 v47, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_b32 v46, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_b32 v45, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_b32 v44, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_b32 v43, off, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s32 offset:68
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v32 :: v_dual_mov_b32 v1, v33
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v34 :: v_dual_mov_b32 v3, v35
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v36 :: v_dual_mov_b32 v5, v37
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v38 :: v_dual_mov_b32 v7, v39
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, v48 :: v_dual_mov_b32 v17, v49
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, v50 :: v_dual_mov_b32 v19, v51
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v20, v52 :: v_dual_mov_b32 v21, v53
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v22, v54 :: v_dual_mov_b32 v23, v55
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -97340,22 +97448,64 @@ define <16 x i64> @bitcast_v64bf16_to_v16i64(<64 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v64bf16_to_v16i64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_clause 0xf
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v40, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v42, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v43, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v44, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v45, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v46, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v47, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v56, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v57, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v58, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v59, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v60, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v61, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v62, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v63, s32 offset:8
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: scratch_load_b32 v32, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v32
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v32
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB62_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v63, v31 :: v_dual_mov_b32 v62, v30
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v61, v29 :: v_dual_mov_b32 v60, v28
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v59, v27 :: v_dual_mov_b32 v58, v26
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v57, v25 :: v_dual_mov_b32 v56, v24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v55, v23 :: v_dual_mov_b32 v54, v22
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v53, v21 :: v_dual_mov_b32 v52, v20
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v51, v19 :: v_dual_mov_b32 v50, v18
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v49, v17 :: v_dual_mov_b32 v48, v16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v47, v15 :: v_dual_mov_b32 v46, v14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v45, v13 :: v_dual_mov_b32 v44, v12
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v43, v11 :: v_dual_mov_b32 v42, v10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v41, v9 :: v_dual_mov_b32 v40, v8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, v7 :: v_dual_mov_b32 v38, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v37, v5 :: v_dual_mov_b32 v36, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, v3 :: v_dual_mov_b32 v34, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, v1 :: v_dual_mov_b32 v32, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: .LBB62_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB62_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v33, 0x40c00000, v33 :: v_dual_lshlrev_b32 v32, 16, v15
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v15
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v33, 0x40c00000, v33 :: v_dual_add_f32 v32, 0x40c00000, v32
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v33, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v32, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v32
@@ -97363,539 +97513,551 @@ define <16 x i64> @bitcast_v64bf16_to_v16i64(<64 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v32, 0x7fff
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v32, v35, v38 :: v_dual_and_b32 v15, 0xffff0000, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_dual_add_f32 v15, 0x40c00000, v15 :: v_dual_lshlrev_b32 v14, 16, v14
; GFX11-TRUE16-NEXT: v_add3_u32 v35, v39, v33, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v32.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v15, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v15
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v36, v36, v15, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_cndmask_b32 v15, v36, v37
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_cndmask_b32 v47, v36, v37
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_bfe_u32 v48, v14, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v49, 0x400000, v14
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v33
-; GFX11-TRUE16-NEXT: v_bfi_b32 v15, 0xffff, v32, v15
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v47.l, v32.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v32, 0x400000, v33
; GFX11-TRUE16-NEXT: v_add3_u32 v38, v48, v14, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v38, v49, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v14.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v33, v35, v36 :: v_dual_add_f32 v34, 0x40c00000, v34
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v14, 0xffff, v14, v33
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v34, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v38, v13, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v34, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v36, v38, v13, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v13
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v13, v36, v37 :: v_dual_and_b32 v38, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v13.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v35, v39, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v13, 0xffff, v13, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v15, 0x40c00000, v34 :: v_dual_cndmask_b32 v46, v35, v32
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v15, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v46.l, v14.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v12
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v36
+; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v13, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v33, v34, v15, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v34, 0x400000, v13
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v35, 0x400000, v15
+; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v13, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v32, v34, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_cndmask_b32 v45, v33, v35
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v45.l, v13.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v12, 0x40c00000, v12 :: v_dual_lshlrev_b32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v14, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v34, 0x400000, v14
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_add_f32 v11, 0x40c00000, v11
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v12, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v35, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v12
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v12, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v32, v35, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v11, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v14, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v12, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v15, v33, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v13, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v44, v32, v34, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v32, 0x400000, v11
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v11.h
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v33, v33, v38 :: v_dual_and_b32 v38, 0xffff0000, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v44.l, v12.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v13, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v11, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v11, v14, v32 :: v_dual_and_b32 v12, 0xffff0000, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v11, 0xffff, v11, v33
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v10, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v12, 0x40c00000, v12 :: v_dual_cndmask_b32 v43, v15, v33
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v43.l, v11.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v10, 0x40c00000, v10 :: v_dual_lshlrev_b32 v9, 16, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v32, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v10
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v10, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v12.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v10.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: v_bfi_b32 v12, 0xffff, v12, v32
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v12, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v10, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v13, v15, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v42, v14, v32, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v9
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v10, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v42.l, v10.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v9, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v12, v14 :: v_dual_and_b32 v10, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v9, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_cndmask_b32 v9, v32, v36
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_add_f32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v10, 0x40c00000, v10 :: v_dual_cndmask_b32 v41, v13, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v41.l, v9.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_lshlrev_b32 v7, 16, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v8
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v9.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v8, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v11, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v40, v12, v14, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v9, 0xffff, v9, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v40.l, v8.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v10, v12 :: v_dual_and_b32 v8, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_bfi_b32 v8, 0xffff, v8, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_cndmask_b32 v39, v11, v13
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v7.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v7.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v7, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_add_f32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v32, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v8, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v38, v10, v12, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v6, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v6.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v7, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v8, v10 :: v_dual_and_b32 v6, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v37, v9, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v5, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v36, v8, v10, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v4.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v6, v8 :: v_dual_and_b32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v34, 0x40c00000, v34
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_cndmask_b32 v3, v32, v36
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_add_f32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_cndmask_b32 v35, v7, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v3.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v2.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v4, v6 :: v_dual_and_b32 v2, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v35, 0x40c00000, v38 :: v_dual_add_f32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v34
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v31
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_add_f32 v31, 0x40c00000, v31
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v32, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v31, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v30
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v31, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v31
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v31, v31
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v31, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v30, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v31.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v30, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v30
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v29
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; GFX11-TRUE16-NEXT: v_bfi_b32 v31, 0xffff, v31, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v30, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v30.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v29, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v30, 0xffff, v30, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v28
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v29, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v29
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v34, 0x40c00000, v34
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v29, v32, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v28, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v28, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v28
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_add_f32 v27, 0x40c00000, v27
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v29.h
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_cndmask_b32 v33, v5, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v28, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v27, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v29, 0xffff, v29, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v28.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v27, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v27
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; GFX11-TRUE16-NEXT: v_bfi_b32 v28, 0xffff, v28, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v27, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v27.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v26, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v27, 0xffff, v27, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v26, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v26
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_add_f32 v25, 0x40c00000, v25
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v26, v32, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v1.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v31
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v5, 16, v31
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v4, v7, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v25, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v24
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v25, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v25
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v26.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v25, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v24, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v26, 0xffff, v26, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v25.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v24, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v24
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX11-TRUE16-NEXT: v_bfi_b32 v25, 0xffff, v25, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v24, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v24.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v23, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v24, 0xffff, v24, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v23, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v23
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v34, 0x40c00000, v34
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v20
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v23, v32, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v22, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v21
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v22, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v22
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_add_f32 v21, 0x40c00000, v21
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v23.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v22, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v21, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v23, 0xffff, v23, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v22.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v21, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v21
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v36
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v22, 0xffff, v22, v33
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v20, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v21, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v39
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v20
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v20, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v21.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v35, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v18
-; GFX11-TRUE16-NEXT: v_bfi_b32 v21, 0xffff, v21, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v33, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_add_f32 v19, 0x40c00000, v19
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v32, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v19, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v19
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v36, v36, v19, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v35, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v20.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v19, v36, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v39, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v18, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v19.h
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v33, v33, v38 :: v_dual_and_b32 v38, 0xffff0000, v17
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v18, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfi_b32 v20, 0xffff, v20, v32
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v37, 0x40c00000, v38 :: v_dual_cndmask_b32 v34, v34, v36
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v16
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v48, 0x400000, v37
-; GFX11-TRUE16-NEXT: v_bfi_b32 v19, 0xffff, v19, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v35, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_bfe_u32 v38, v17, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v49, 0x400000, v17
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v37, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v38, v38, v17, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v51, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v37, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v39, v39, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v38, v49, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_bfe_u32 v50, v16, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v49, 0x400000, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v18.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v17.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v36, v39, v51, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX11-TRUE16-NEXT: v_add3_u32 v38, v50, v16, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfi_b32 v18, 0xffff, v18, v34
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v36.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v35, v35, v48, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT: v_bfi_b32 v17, 0xffff, v17, v35
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v38, v49, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v0.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v63, v3, v7 :: v_dual_and_b32 v0, 0xffff0000, v30
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v29
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v30
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v16, 0xffff, v36, v16
-; GFX11-TRUE16-NEXT: .LBB62_2: ; %end
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v62, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v62.l, v1.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v28
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v63.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v29
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v61, v3, v7 :: v_dual_lshlrev_b32 v4, 16, v28
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v27
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v61.l, v0.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v27
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v60, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v59, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v59.l, v1.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v25
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v60.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v26
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v25
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v4, 16, v26
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v58, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v4, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v24
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v58.l, v0.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v24
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v57, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v57.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v23
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v23
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_add_f32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v56, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v56.l, v1.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v22
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v22
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v55, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v21
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v0.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v21
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v5, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v54, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v2.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v20
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v20
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v5, 16, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v53, v2, v7, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v1.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v19
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v6, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v52, v0, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v4, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v3, v4 :: v_dual_lshlrev_b32 v6, 16, v18
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v6 :: v_dual_cndmask_b32 v51, v0, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v3.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v17
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v18
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v8, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v50, v0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v5, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v16
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v0, v1 :: v_dual_add_f32 v1, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v3, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v0.h
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v9, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v49, v2, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v0.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v48, v1, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v5.h
+; GFX11-TRUE16-NEXT: .LBB62_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v40 :: v_dual_mov_b32 v9, v41
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, v42 :: v_dual_mov_b32 v11, v43
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v44 :: v_dual_mov_b32 v13, v45
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, v46 :: v_dual_mov_b32 v15, v47
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v24, v56 :: v_dual_mov_b32 v25, v57
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, v58 :: v_dual_mov_b32 v27, v59
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v28, v60 :: v_dual_mov_b32 v29, v61
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v30, v62 :: v_dual_mov_b32 v31, v63
+; GFX11-TRUE16-NEXT: s_clause 0xf
+; GFX11-TRUE16-NEXT: scratch_load_b32 v63, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_b32 v62, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_b32 v61, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_b32 v60, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_b32 v59, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_b32 v58, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_b32 v57, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_b32 v56, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_b32 v47, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_b32 v46, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_b32 v45, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_b32 v44, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_b32 v43, off, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s32 offset:68
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v32 :: v_dual_mov_b32 v1, v33
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v34 :: v_dual_mov_b32 v3, v35
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v36 :: v_dual_mov_b32 v5, v37
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v38 :: v_dual_mov_b32 v7, v39
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, v48 :: v_dual_mov_b32 v17, v49
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, v50 :: v_dual_mov_b32 v19, v51
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v20, v52 :: v_dual_mov_b32 v21, v53
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v22, v54 :: v_dual_mov_b32 v23, v55
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -131939,22 +132101,64 @@ define <16 x double> @bitcast_v64bf16_to_v16f64(<64 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v64bf16_to_v16f64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_clause 0xf
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v40, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v42, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v43, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v44, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v45, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v46, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v47, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v56, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v57, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v58, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v59, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v60, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v61, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v62, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v63, s32 offset:8
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: scratch_load_b32 v32, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v32
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v32
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47_vgpr48_vgpr49_vgpr50_vgpr51_vgpr52_vgpr53_vgpr54_vgpr55_vgpr56_vgpr57_vgpr58_vgpr59_vgpr60_vgpr61_vgpr62_vgpr63
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB78_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v63, v31 :: v_dual_mov_b32 v62, v30
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v61, v29 :: v_dual_mov_b32 v60, v28
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v59, v27 :: v_dual_mov_b32 v58, v26
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v57, v25 :: v_dual_mov_b32 v56, v24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v55, v23 :: v_dual_mov_b32 v54, v22
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v53, v21 :: v_dual_mov_b32 v52, v20
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v51, v19 :: v_dual_mov_b32 v50, v18
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v49, v17 :: v_dual_mov_b32 v48, v16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v47, v15 :: v_dual_mov_b32 v46, v14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v45, v13 :: v_dual_mov_b32 v44, v12
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v43, v11 :: v_dual_mov_b32 v42, v10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v41, v9 :: v_dual_mov_b32 v40, v8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, v7 :: v_dual_mov_b32 v38, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v37, v5 :: v_dual_mov_b32 v36, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, v3 :: v_dual_mov_b32 v34, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, v1 :: v_dual_mov_b32 v32, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: .LBB78_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB78_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v33, 0x40c00000, v33 :: v_dual_lshlrev_b32 v32, 16, v15
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v15
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v33, 0x40c00000, v33 :: v_dual_add_f32 v32, 0x40c00000, v32
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v33, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v32, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v32
@@ -131962,539 +132166,551 @@ define <16 x double> @bitcast_v64bf16_to_v16f64(<64 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v32, 0x7fff
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v32, v35, v38 :: v_dual_and_b32 v15, 0xffff0000, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_dual_add_f32 v15, 0x40c00000, v15 :: v_dual_lshlrev_b32 v14, 16, v14
; GFX11-TRUE16-NEXT: v_add3_u32 v35, v39, v33, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v32.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v15, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v15
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v36, v36, v15, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_cndmask_b32 v15, v36, v37
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_cndmask_b32 v47, v36, v37
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_bfe_u32 v48, v14, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v49, 0x400000, v14
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v33
-; GFX11-TRUE16-NEXT: v_bfi_b32 v15, 0xffff, v32, v15
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v47.l, v32.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v32, 0x400000, v33
; GFX11-TRUE16-NEXT: v_add3_u32 v38, v48, v14, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v38, v49, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v14.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v33, v35, v36 :: v_dual_add_f32 v34, 0x40c00000, v34
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v14, 0xffff, v14, v33
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v34, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v38, v13, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v34, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v36, v38, v13, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v13
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v13, v36, v37 :: v_dual_and_b32 v38, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v13.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v35, v39, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v13, 0xffff, v13, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v15, 0x40c00000, v34 :: v_dual_cndmask_b32 v46, v35, v32
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v15, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v46.l, v14.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v12
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v36
+; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v13, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v33, v34, v15, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v34, 0x400000, v13
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v35, 0x400000, v15
+; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v13, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v32, v34, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_cndmask_b32 v45, v33, v35
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v45.l, v13.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v12, 0x40c00000, v12 :: v_dual_lshlrev_b32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v14, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v34, 0x400000, v14
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_add_f32 v11, 0x40c00000, v11
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v12, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v35, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v12
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v12, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v32, v35, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v11, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v14, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v12, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v15, v33, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v13, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v44, v32, v34, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v32, 0x400000, v11
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v11.h
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v33, v33, v38 :: v_dual_and_b32 v38, 0xffff0000, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v44.l, v12.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v13, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v11, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v11, v14, v32 :: v_dual_and_b32 v12, 0xffff0000, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v11, 0xffff, v11, v33
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v10, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v12, 0x40c00000, v12 :: v_dual_cndmask_b32 v43, v15, v33
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v43.l, v11.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v10, 0x40c00000, v10 :: v_dual_lshlrev_b32 v9, 16, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v32, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v10
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v10, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v12.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v10.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: v_bfi_b32 v12, 0xffff, v12, v32
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v12, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v10, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v13, v15, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v42, v14, v32, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v9
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v10, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v42.l, v10.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v9, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v12, v14 :: v_dual_and_b32 v10, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v9, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_cndmask_b32 v9, v32, v36
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_add_f32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v10, 0x40c00000, v10 :: v_dual_cndmask_b32 v41, v13, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v41.l, v9.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_lshlrev_b32 v7, 16, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v8
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v9.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v8, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v11, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v40, v12, v14, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v9, 0xffff, v9, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v40.l, v8.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v10, v12 :: v_dual_and_b32 v8, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_bfi_b32 v8, 0xffff, v8, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_cndmask_b32 v39, v11, v13
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v7.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v7.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v7, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_add_f32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v32, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v8, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v38, v10, v12, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v6, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v6.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v7, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v8, v10 :: v_dual_and_b32 v6, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v37, v9, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v5, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v36, v8, v10, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v4.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v6, v8 :: v_dual_and_b32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v34, 0x40c00000, v34
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_cndmask_b32 v3, v32, v36
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_add_f32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_cndmask_b32 v35, v7, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v3.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v2.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v4, v6 :: v_dual_and_b32 v2, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v35, 0x40c00000, v38 :: v_dual_add_f32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v34
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v31
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_add_f32 v31, 0x40c00000, v31
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v32, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v31, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v30
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v31, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v31
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v31, v31
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v31, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v30, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v31.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v30, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v30
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v29
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; GFX11-TRUE16-NEXT: v_bfi_b32 v31, 0xffff, v31, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v30, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v30.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v29, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v30, 0xffff, v30, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v28
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v29, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v29
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v34, 0x40c00000, v34
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v29, v32, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v28, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v28, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v28
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_add_f32 v27, 0x40c00000, v27
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v29.h
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_cndmask_b32 v33, v5, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v28, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v27, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v29, 0xffff, v29, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v28.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v27, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v27
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; GFX11-TRUE16-NEXT: v_bfi_b32 v28, 0xffff, v28, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v27, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v27.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v26, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v27, 0xffff, v27, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v26, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v26
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_add_f32 v25, 0x40c00000, v25
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v26, v32, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v1.h
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v31
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v5, 16, v31
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v4, v7, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v25, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v24
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v25, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v25
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v26.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v25, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v24, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v26, 0xffff, v26, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v25.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v24, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v24
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX11-TRUE16-NEXT: v_bfi_b32 v25, 0xffff, v25, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v24, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v24.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v23, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v24, 0xffff, v24, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v23, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v23
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v34, 0x40c00000, v34
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v20
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v23, v32, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v22, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v21
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v22, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v22
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_add_f32 v21, 0x40c00000, v21
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v23.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v22, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v21, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v23, 0xffff, v23, v32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v22.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v35, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v21, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v21
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v36
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v22, 0xffff, v22, v33
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v20, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v21, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v39
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v20
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v20, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v21.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v35, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v18
-; GFX11-TRUE16-NEXT: v_bfi_b32 v21, 0xffff, v21, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v33, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_add_f32 v19, 0x40c00000, v19
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v32, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v19, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v19
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v36, v36, v19, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v35, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v20.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v19, v36, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v39, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v18, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v19.h
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v33, v33, v38 :: v_dual_and_b32 v38, 0xffff0000, v17
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v18, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfi_b32 v20, 0xffff, v20, v32
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v37, 0x40c00000, v38 :: v_dual_cndmask_b32 v34, v34, v36
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v16
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v16
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v48, 0x400000, v37
-; GFX11-TRUE16-NEXT: v_bfi_b32 v19, 0xffff, v19, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v35, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_bfe_u32 v38, v17, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v49, 0x400000, v17
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v37, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v38, v38, v17, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v51, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v37, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v39, v39, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v38, v49, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_bfe_u32 v50, v16, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v49, 0x400000, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v18.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v17.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v36, v39, v51, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX11-TRUE16-NEXT: v_add3_u32 v38, v50, v16, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfi_b32 v18, 0xffff, v18, v34
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v36.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v35, v35, v48, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT: v_bfi_b32 v17, 0xffff, v17, v35
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v38, v49, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v0.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v63, v3, v7 :: v_dual_and_b32 v0, 0xffff0000, v30
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v29
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v30
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v16, 0xffff, v36, v16
-; GFX11-TRUE16-NEXT: .LBB78_2: ; %end
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v62, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v62.l, v1.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v28
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v63.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v29
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v61, v3, v7 :: v_dual_lshlrev_b32 v4, 16, v28
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v27
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v61.l, v0.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v27
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v60, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v59, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v59.l, v1.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v25
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v60.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v26
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v25
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v4, 16, v26
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v58, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v4, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v24
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v58.l, v0.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v24
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v57, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v57.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v23
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v23
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_add_f32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v56, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v56.l, v1.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v22
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v22
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v55, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v21
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v0.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v21
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v5, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v54, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v2.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v20
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v20
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v5, 16, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v53, v2, v7, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v1.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v19
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v6, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v52, v0, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v4, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v3, v4 :: v_dual_lshlrev_b32 v6, 16, v18
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v6 :: v_dual_cndmask_b32 v51, v0, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v3.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v17
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v18
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v8, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v50, v0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v5, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v16
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v0, v1 :: v_dual_add_f32 v1, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v3, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v0.h
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v9, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v49, v2, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v0.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v48, v1, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v5.h
+; GFX11-TRUE16-NEXT: .LBB78_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v40 :: v_dual_mov_b32 v9, v41
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, v42 :: v_dual_mov_b32 v11, v43
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v44 :: v_dual_mov_b32 v13, v45
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, v46 :: v_dual_mov_b32 v15, v47
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v24, v56 :: v_dual_mov_b32 v25, v57
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, v58 :: v_dual_mov_b32 v27, v59
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v28, v60 :: v_dual_mov_b32 v29, v61
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v30, v62 :: v_dual_mov_b32 v31, v63
+; GFX11-TRUE16-NEXT: s_clause 0xf
+; GFX11-TRUE16-NEXT: scratch_load_b32 v63, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_b32 v62, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_b32 v61, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_b32 v60, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_b32 v59, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_b32 v58, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_b32 v57, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_b32 v56, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_b32 v47, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_b32 v46, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_b32 v45, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_b32 v44, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_b32 v43, off, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s32 offset:68
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v32 :: v_dual_mov_b32 v1, v33
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v34 :: v_dual_mov_b32 v3, v35
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v36 :: v_dual_mov_b32 v5, v37
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v38 :: v_dual_mov_b32 v7, v39
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, v48 :: v_dual_mov_b32 v17, v49
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, v50 :: v_dual_mov_b32 v19, v51
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v20, v52 :: v_dual_mov_b32 v21, v53
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v22, v54 :: v_dual_mov_b32 v23, v55
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -159577,1208 +159793,1232 @@ define <128 x i8> @bitcast_v64bf16_to_v128i8(<64 x bfloat> %a, i32 %b) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v40, s32 offset:168
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s32 offset:164
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v42, s32 offset:160
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v43, s32 offset:156
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v44, s32 offset:152
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v45, s32 offset:148
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v46, s32 offset:144
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v47, s32 offset:140
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v56, s32 offset:136
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v57, s32 offset:132
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v58, s32 offset:128
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v59, s32 offset:124
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v60, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v61, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v62, s32 offset:112
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v63, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v72, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v73, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v74, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v75, s32 offset:92
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v76, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v77, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v78, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v79, s32 offset:76
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v88, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v89, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v90, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v91, s32 offset:60
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v92, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v93, s32 offset:52
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v94, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v95, s32 offset:44
-; GFX11-TRUE16-NEXT: s_clause 0x7
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v104, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v105, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v106, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v107, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v108, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v109, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v110, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v111, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v40, s32 offset:244
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s32 offset:240
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v42, s32 offset:236
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v43, s32 offset:232
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v44, s32 offset:228
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v45, s32 offset:224
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v46, s32 offset:220
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v47, s32 offset:216
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v56, s32 offset:212
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v57, s32 offset:208
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v58, s32 offset:204
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v59, s32 offset:200
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v60, s32 offset:196
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v61, s32 offset:192
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v62, s32 offset:188
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v63, s32 offset:184
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v72, s32 offset:180
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v73, s32 offset:176
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v74, s32 offset:172
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v75, s32 offset:168
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v76, s32 offset:164
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v77, s32 offset:160
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v78, s32 offset:156
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v79, s32 offset:152
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v88, s32 offset:148
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v89, s32 offset:144
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v90, s32 offset:140
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v91, s32 offset:136
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v92, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v93, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v94, s32 offset:124
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v95, s32 offset:120
+; GFX11-TRUE16-NEXT: s_clause 0x1a
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v104, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v105, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v106, s32 offset:108
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v107, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v108, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v109, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v110, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v111, s32 offset:88
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v120, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v121, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v122, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v123, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v124, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v125, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v126, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v127, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v136, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v137, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v138, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v139, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v140, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v141, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v142, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v143, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v152, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v153, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v154, s32 offset:12
; GFX11-TRUE16-NEXT: s_clause 0x2
-; GFX11-TRUE16-NEXT: scratch_load_b32 v33, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_b32 v32, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr111_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr106_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr105_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr104_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr95_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr135_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr93_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr166_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr88_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr78_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr76_lo16
+; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_b32 v85, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_b32 v84, off, s32
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr180_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr143_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr146_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr179_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr142_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr141_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr43_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr74_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr177_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr63_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr60_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr73_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr57_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr41_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr47_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr45_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr92_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr40_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr59_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr140_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr183_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr139_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr127_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr62_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr180_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr108_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr176_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr125_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr133_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr47_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr123_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr121_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr91_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr165_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr89_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr163_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr110_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr107_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr109_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr147_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr110_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr94_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr74_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr107_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr105_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr111_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr95_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr89_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr93_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr90_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr138_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr79_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr108_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr77_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr75_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr153_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr72_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr137_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr61_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr59_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr154_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr57_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr152_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr46_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr44_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr136_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr126_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr124_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr150_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr122_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr149_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr120_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr109_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr106_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr151_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr104_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr94_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr162_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr92_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr161_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr88_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr78_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr76_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr163_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr73_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr63_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr166_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr60_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr165_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr58_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr56_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr46_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr176_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr45_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr167_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr42_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr183_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr41_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr178_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr40_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr177_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr182_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr181_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr179_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr167_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr164_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr162_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr160_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr148_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr145_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr144_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr134_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr132_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v33
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v31
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB90_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[67:68], 24, v[7:8]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[100:101], 24, v[15:16]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[116:117], 24, v[11:12]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[132:133], 24, v[7:8]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[133:134], 24, v[5:6]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[101:102], 24, v[23:24]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[117:118], 24, v[19:20]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v44, 24, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v46, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v57, 8, v15
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v59, 24, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v61, 8, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v72, 8, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v75, 24, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v77, 8, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v79, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v90, 24, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v93, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v95, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v105, 24, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v107, 8, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v110, 8, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v121, 24, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v123, 8, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v125, 8, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v127, 24, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v139, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v140, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v141, 24, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v142, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v143, 8, v1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v181, 24, v85
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v182, 8, v85
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[33:34], 24, v[31:32]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[68:69], 24, v[5:6]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[29:30]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[50:51], 24, v[13:14]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[53:54], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[64:65], 24, v[9:10]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[69:70], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[27:28]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 24, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 8, v15
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v163, 24, v14
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v165, 8, v14
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v176, 8, v13
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v180, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v182, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v40, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v45, 24, v10
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v47, 8, v10
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v57, 8, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v60, 24, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v63, 8, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v74, 8, v7
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v76, 24, v6
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v78, 8, v6
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v88, 8, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v93, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v95, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v104, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v105, 24, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v106, 8, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v111, 8, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 24, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 8, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v134, 8, v31
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v144, 24, v30
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v145, 8, v30
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v148, 8, v29
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 24, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v162, 8, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v164, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v167, 24, v26
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v179, 8, v26
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v181, 8, v25
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v183, 24, v24
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v42, 8, v24
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v46, 8, v23
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v56, 24, v22
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v58, 8, v22
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v61, 8, v21
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v72, 24, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v75, 8, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v77, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v79, 24, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v90, 8, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v94, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[38:39], 24, v[15:16]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[70:71], 24, v[1:2]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[36:37], 24, v[25:26]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[48:49], 24, v[23:24]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[51:52], 24, v[21:22]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[54:55], 24, v[19:20]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[65:66], 24, v[17:18]
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.h, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v128.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v129.h, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v146.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v133.h, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v135.h, v4.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v166.h, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v150.h, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v151.h, v6.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v43.h, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v177.h, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v178.h, v8.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v73.h, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v41.h, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v44.h, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v92.h, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v59.h, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v62.h, v12.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v108.h, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v91.h, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v89.h, v14.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v110.h, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v107.h, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v109.h, v16.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.h, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.h, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.h, v18.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.h, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.h, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v84.h, v20.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.h, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.h, v22.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.h, v22.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v99.h, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.h, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v98.h, v24.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v102.h, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v100.h, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v101.h, v26.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v113.h, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v103.h, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v112.h, v28.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v116.h, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.h, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v115.h, v30.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.h, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v117.h, v32.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v118.h, v32.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v40, 8, v84
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v41, 24, v30
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v42, 8, v30
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v45, 8, v29
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v56, 24, v28
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v58, 8, v28
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v60, 8, v27
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v63, 24, v26
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v73, 8, v26
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v76, 8, v25
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v78, 24, v24
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v88, 8, v24
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v92, 8, v23
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v94, 24, v22
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v104, 8, v22
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v106, 8, v21
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v109, 24, v20
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v120, 8, v20
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v122, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v124, 24, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v126, 8, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v136, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[112:113], 24, v[13:14]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[128:129], 24, v[9:10]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[134:135], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[146:147], 24, v[1:2]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[70:71], 24, v[84:85]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[80:81], 24, v[29:30]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[86:87], 24, v[27:28]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[96:97], 24, v[25:26]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[102:103], 24, v[21:22]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[118:119], 24, v[17:18]
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v180.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.h, v1.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v179.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.h, v2.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v43.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.h, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v183.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.h, v4.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v62.h, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.h, v5.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v47.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.h, v6.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v91.h, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.h, v7.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v74.h, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.h, v8.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v111.h, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v98.h, v9.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v89.h, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v99.h, v10.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v138.h, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.h, v11.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v108.h, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v115.h, v12.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v153.h, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v130.h, v13.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v137.h, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.h, v14.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v154.h, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.h, v15.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v152.h, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v145.h, v16.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v148.h, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.h, v17.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.h, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.h, v18.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v150.h, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.h, v19.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v149.h, v20.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.h, v20.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v160.h, v21.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.h, v21.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v151.h, v22.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v22.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v162.h, v23.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.h, v23.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v161.h, v24.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.h, v24.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v164.h, v25.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.h, v25.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v163.h, v26.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.h, v26.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v166.h, v27.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.h, v27.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v165.h, v28.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.h, v28.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v176.h, v29.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.h, v29.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v167.h, v30.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.h, v30.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v178.h, v84.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.h, v84.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v177.h, v85.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.h, v85.h
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85
; GFX11-TRUE16-NEXT: .LBB90_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB90_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v17
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v18, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v18
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v32 :: v_dual_lshlrev_b32 v31, 16, v18
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v38, v34, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v31, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v31
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v31, v31
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v31, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v18
+; GFX11-TRUE16-NEXT: v_add3_u32 v31, v38, v34, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v39, v32, v37, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v18, 0x40c00000, v18 :: v_dual_lshlrev_b32 v17, 16, v17
+; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v18, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v18
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v37, v37, v18, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v80, v37, v39 :: v_dual_add_f32 v33, 0x40c00000, v33
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v33, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v33
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX11-TRUE16-NEXT: v_add3_u32 v36, v36, v33, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v17
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v80.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v81, v36, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v34, 0x40c00000, v34 :: v_dual_lshlrev_b32 v17, 16, v17
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX11-TRUE16-NEXT: v_bfe_u32 v48, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v49, 0x400000, v34
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v50, v17, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v51, 0x400000, v17
+; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v18, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v17, 0x40c00000, v17 :: v_dual_cndmask_b32 v32, v35, v36
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v48, v17, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v49, 0x400000, v17
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v48, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v37, v50, v17, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v82, v37, v51 :: v_dual_and_b32 v35, 0xffff0000, v20
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v35, 0x40c00000, v35 :: v_dual_lshlrev_b32 v20, 16, v20
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v39.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v37, v48, v17, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v20
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v124, 24, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v126, 8, v32
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v148, v37, v49, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v33
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v34
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v82.h
; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v35, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v18, v49, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v18, 0xffff, v33, v81
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v35
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v19
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v31, v31, v33, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v20, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v20
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v35, 0x400000, v20
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v34 :: v_dual_add_f32 v19, 0x40c00000, v19
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_bfi_b32 v17, 0xffff, v34, v17
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v36, v35, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v20, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v79, 24, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v90, 8, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v83, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_lshlrev_b32 v19, 16, v19
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v22
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v19, 0x40c00000, v19 :: v_dual_lshlrev_b32 v22, 16, v22
+; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v18, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v34, 0x400000, v18
; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v84, v34, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v83.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v19, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v19
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v31.l, v148.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v149, v33, v35, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v35, 0x400000, v36
; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v94, 8, v17
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v19, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v22, 0x40c00000, v22 :: v_dual_cndmask_b32 v85, v33, v37
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v22, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v19
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v17, v34, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v19, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v34.l, v149.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v136, 8, v31
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v19, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v150, v17, v33, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v85.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v36, v20, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v20, v33, v22, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v22
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v86, v20, v33 :: v_dual_add_f32 v35, 0x40c00000, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v20, 0xffff, v34, v84
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v86.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v35, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v72, 24, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v75, 8, v20
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v21
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v33, v20, v35 :: v_dual_and_b32 v20, 0xffff0000, v21
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v87, v19, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v19, 0xffff, v37, v36
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v24
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v33, 0x40c00000, v38 :: v_dual_lshlrev_b32 v24, 16, v24
-; GFX11-TRUE16-NEXT: v_bfi_b32 v22, 0xffff, v22, v87
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v109, 24, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v120, 8, v34
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v21, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v33, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v21
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v24, 0x40c00000, v24
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v21, 0x40c00000, v21 :: v_dual_lshlrev_b32 v22, 16, v22
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v22, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v35, 0x400000, v22
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v22, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v151, v19, v35 :: v_dual_lshlrev_b32 v22, 16, v24
+; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v20, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v35, 0x400000, v20
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v20, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v18
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v18, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v24
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v21
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v36, v17, v36, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v21, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v34, v21, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v33, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v33
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v56, 24, v22
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v96, v34, v37 :: v_dual_and_b32 v37, 0xffff0000, v23
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v24, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v58, 8, v22
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v21, v35, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v24
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v34, v24, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v23, 0x40c00000, v23
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v96.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v77, 8, v19
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v97, v34, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v37, 0x40c00000, v37
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v23, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v98, v33, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v24, v37, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v23
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v34, v23, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v37
-; GFX11-TRUE16-NEXT: v_add3_u32 v24, v24, v37, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
-; GFX11-TRUE16-NEXT: v_bfi_b32 v21, 0xffff, v35, v21
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v99, v34, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v34, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v61, 8, v21
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v99.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v23, v24, v39, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v23, 0xffff, v36, v23
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v25
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v97.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v26
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT: v_bfi_b32 v24, 0xffff, v33, v98
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v26, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v25, 0x40c00000, v25
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v46, 8, v23
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v183, 24, v24
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v26, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v26, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v42, 8, v24
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v100, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v25, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v25
-; GFX11-TRUE16-NEXT: v_add3_u32 v26, v26, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v101, v35, v38, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v25, 0x7fff
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v28
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v100.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v102, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v35, 0x40c00000, v35 :: v_dual_lshlrev_b32 v28, 16, v28
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v102.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
-; GFX11-TRUE16-NEXT: v_bfe_u32 v25, v35, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v36, v26, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v28, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-TRUE16-NEXT: v_add3_u32 v25, v25, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
-; GFX11-TRUE16-NEXT: v_add3_u32 v26, v33, v28, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v28
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v36.l, v151.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v21, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v103, v26, v33, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v26, 0xffff, v34, v101
-; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v27, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v33, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v103.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v112, v25, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v25, 0xffff, v37, v36
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v34, v27, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v27
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v30
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v33, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v33
-; GFX11-TRUE16-NEXT: v_bfi_b32 v28, 0xffff, v28, v112
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v113, v34, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff0000, v29
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v33, 0x7fff
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_add_f32 v37, 0x40c00000, v37
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v30, 0x40c00000, v30 :: v_dual_lshlrev_b32 v29, 16, v29
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v27, v35, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v160, v17, v24 :: v_dual_lshlrev_b32 v21, 16, v23
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v23
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v22
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v18
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v35, v19, v35, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v22, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v18, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v21, 0x40c00000, v21 :: v_dual_add_f32 v20, 0x40c00000, v20
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v22, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v26
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v94, 24, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v104, 8, v36
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v161, v19, v23, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v20, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v21
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v26
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v38, v17, v24, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v21, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v20, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v20
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
+; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v21, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v25
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v33.l, v150.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v38.l, v161.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v30, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v30
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v29, 0x40c00000, v29
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v34, v30, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v30, v37, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v113.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 24, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v162, 8, v28
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v114, v34, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v162, v17, v23, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v22
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v25
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v37, v19, v24, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v22, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v18, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v18
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
+; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v22, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v28
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v122, 8, v33
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v163, v19, v23, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v20, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v21
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v28
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v22, 0x40c00000, v22 :: v_dual_cndmask_b32 v49, v17, v24
+; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v21, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v20, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v20
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
+; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v21, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v27
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v35.l, v160.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v49.l, v163.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v164, v17, v23, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v22
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v27
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v21, 0x40c00000, v21 :: v_dual_cndmask_b32 v48, v19, v24
+; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v22, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v18, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v18
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
+; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v22, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v30
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v63, 24, v49
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v73, 8, v49
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v106, 8, v35
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v165, v19, v23, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v20, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v21
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v30
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v22, 0x40c00000, v22 :: v_dual_cndmask_b32 v51, v17, v24
+; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v21, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v20, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v20
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
+; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v21, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v29
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v51.l, v165.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v78, 24, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v88, 8, v38
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v166, v17, v23, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v22
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v29
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v21, 0x40c00000, v21 :: v_dual_cndmask_b32 v50, v19, v24
+; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v22, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v18, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v18
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
+; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v22, 0x7fff
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v32
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v32
-; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v29, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v29
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v115, v33, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11-TRUE16-NEXT: v_add3_u32 v30, v30, v37, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v34, v29, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v37
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v114.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v27, 0xffff, v35, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v167, 24, v26
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v116, v34, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v34, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v164, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v179, 8, v26
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v116.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v29, v30, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_bfi_b32 v30, 0xffff, v33, v115
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v181, 8, v25
-; GFX11-TRUE16-NEXT: v_bfi_b32 v29, 0xffff, v36, v29
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v85
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v56, 24, v51
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v58, 8, v51
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v167, v19, v23, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v20, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v21
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v85
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v22, 0x40c00000, v22 :: v_dual_cndmask_b32 v53, v17, v24
+; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v21, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v20, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v20
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
+; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v21, 0x7fff
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v31
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v32, 0x40c00000, v32 :: v_dual_lshlrev_b32 v31, 16, v31
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v144, 24, v30
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v145, 8, v30
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v32, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v32
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v148, 8, v29
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v32, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v117, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v31, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v31
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v117.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v31, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v118, v35, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v31, v31
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v35
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v84
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v53.l, v167.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v119, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_bfe_u32 v31, v35, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v119.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v31, v31, v35, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v176, v17, v23, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v22
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v84
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v21, 0x40c00000, v21 :: v_dual_cndmask_b32 v52, v19, v24
+; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v22, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v18, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v18
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
+; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v22, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v21
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v37.l, v162.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v41, 24, v53
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v42, 8, v53
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v177, v19, v23, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v20, 16, 1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v36, v32, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v20
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v55, v17, v24, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v21, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v20, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
+; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v21, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v55.l, v177.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v92, 8, v37
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v178, v17, v22, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v18
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v54, v19, v23, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v2, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v33, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v128, v32, v33, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_bfi_b32 v32, 0xffff, v34, v118
-; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v33, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v2.l, v128.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v129, v31, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v31, 0xffff, v37, v36
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v34, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v18, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v20, 0x40c00000, v20
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v181, 24, v55
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v182, 8, v55
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v20, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v179, v19, v21, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v4
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v33, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v131, v34, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v20, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v65, v17, v22, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v20
; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v33, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v35, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
+; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v65.l, v179.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v18, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v180, v17, v19, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v18, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v64, v2, v21, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v34, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v35.l, v131.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v129
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v133, v34, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v18
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v35, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v105, 24, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v135, v33, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v48.l, v164.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v141, 24, v65
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v183, v2, v19, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v17, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v17
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v67, v1, v20
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v3, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v17, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v67.l, v183.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v50.l, v166.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[96:97], 24, v[48:49]
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v43, v1, v18, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v66, v2, v19, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v43.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[86:87], 24, v[50:51]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v127, 24, v67
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v139, 8, v67
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v47, v2, v17, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v34, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v106, 8, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v111, 8, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 24, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 8, v32
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v146, v34, v36 :: v_dual_add_f32 v37, 0x40c00000, v37
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v134, 8, v31
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v36.l, v146.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v37, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v37
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v37, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v4, v39 :: v_dual_add_f32 v34, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v69, v1, v18, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v47.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v62, v2, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v121, 24, v69
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v68, v1, v17, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v2, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v36, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v33.l, v133.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v123, 8, v69
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v74, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v10
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v83, v1, v8
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.l, v74.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v52.l, v176.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v82, v1, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v91, v4, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v12
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v91.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v89, v1, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v9
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v33, v135
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v12
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v93, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v95, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v104, 8, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v150, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v151, v35, v38, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v99, v2, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v13
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v54.l, v178.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v99.l, v89.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[132:133], 24, v[82:83]
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v98, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v7, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[70:71], 24, v[54:55]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[80:81], 24, v[52:53]
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v6, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v111, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v8
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v8
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v34.l, v150.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v166, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v35, 0x40c00000, v35 :: v_dual_lshlrev_b32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v37.l, v166.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v35, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v36, v6, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v35
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v33, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v177, v6, v33, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v33, 0x40c00000, v39
-; GFX11-TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v34, v151
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v178, v5, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v37, v36
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v7 :: v_dual_lshlrev_b32 v36, 16, v10
-; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v33, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v33
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v8.l, v177.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v36
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v34, v33, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfi_b32 v8, 0xffff, v8, v178
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v76, 24, v6
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v60, 24, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v43, v35, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v63, 8, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v78, 8, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v34, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v43.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v88, 8, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v41, v7, v37, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v10, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v10
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v10, 0x7fff
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v12
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v44, v35, v38 :: v_dual_and_b32 v39, 0xffff0000, v9
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v37, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v41.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v38, v37, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v50, 0x400000, v37
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX11-TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v35, v44
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v38, v38, v37, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v45, 24, v10
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v47, 8, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v59, v38, v50, vcc_lo
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v12, 0x40c00000, v12 :: v_dual_lshlrev_b32 v7, 16, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v37, 0x40c00000, v51 :: v_dual_lshlrev_b32 v14, 16, v14
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; GFX11-TRUE16-NEXT: v_bfe_u32 v48, v12, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v52, 0x400000, v12
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v49, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_add3_u32 v48, v48, v12, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v59.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v73, v35, v49, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v39
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v37, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v49, v14, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v62, v48, v52, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_bfi_b32 v12, 0xffff, v7, v62
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v36, v36, v9, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v11
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v35, v37, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v35, 0x400000, v37
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v180, 24, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v36, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v182, 8, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v11, v35, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v38
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v37, v39, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_bfe_u32 v48, v35, 16, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v39 :: v_dual_cndmask_b32 v92, v37, v38
-; GFX11-TRUE16-NEXT: v_add3_u32 v37, v48, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_add3_u32 v39, v49, v14, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v48, 0x400000, v14
-; GFX11-TRUE16-NEXT: v_bfe_u32 v49, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v89, v37, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v35, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_add3_u32 v14, v49, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v37, 0xffff0000, v16
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v91, v39, v48, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v13, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v7, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v14
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v98.l, v111.h
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v108, v1, v2 :: v_dual_lshlrev_b32 v1, 16, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[128:129], 24, v[98:99]
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v115, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v115.l, v108.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v90, 24, v99
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v114, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v4 :: v_dual_add_f32 v3, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v73.h
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v14, v35 :: v_dual_add_f32 v14, 0x40c00000, v37
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v15
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v39, v13, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v13
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v16, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v37, 0x40c00000, v37
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v49, 0x400000, v16
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v108, v35, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v16, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v37, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT: v_bfe_u32 v48, v14, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v51, 0x400000, v37
-; GFX11-TRUE16-NEXT: v_add3_u32 v39, v39, v37, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v107, v13, v49, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v48, v14, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v48, 0x400000, v14
-; GFX11-TRUE16-NEXT: v_bfe_u32 v50, v15, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v110, v39, v51, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v108.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v13, v50, v15, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v91.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v92.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v109, v35, v48, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v110.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v107.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v14, 0xffff, v38, v89
-; GFX11-TRUE16-NEXT: v_bfi_b32 v11, 0xffff, v39, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v13, v16, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v9, 0xffff, v36, v9
-; GFX11-TRUE16-NEXT: v_bfi_b32 v16, 0xffff, v35, v109
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[48:49], 24, v[23:24]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[53:54], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_bfi_b32 v15, 0xffff, v15, v13
-; GFX11-TRUE16-NEXT: v_bfi_b32 v13, 0xffff, v37, v7
-; GFX11-TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v34, v33
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[33:34], 24, v[31:32]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[29:30]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[64:65], 24, v[9:10]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[50:51], 24, v[13:14]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[67:68], 24, v[7:8]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[68:69], 24, v[5:6]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[69:70], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[27:28]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[38:39], 24, v[15:16]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[70:71], 24, v[1:2]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[36:37], 24, v[25:26]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[51:52], 24, v[21:22]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[54:55], 24, v[19:20]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[65:66], 24, v[17:18]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 24, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v149, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v161, 8, v15
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v163, 24, v14
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v165, 8, v14
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v176, 8, v13
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v40, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v57, 8, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v74, 8, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v64.l, v180.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v138, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v131, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v8, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v16
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v137, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v15
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v114.l, v138.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v130, v3, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v16
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v62.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.l, v137.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v153, v4, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v130.l, v153.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v152, v2, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[116:117], 24, v[114:115]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[133:134], 24, v[68:69]
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v154, v7, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v10, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[117:118], 24, v[33:34]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[112:113], 24, v[130:131]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[134:135], 24, v[66:67]
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v145, v4, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v145.l, v152.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[146:147], 24, v[64:65]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[118:119], 24, v[31:32]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v59, 24, v131
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v144, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e64 v144.l, v154.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v44, 24, v145
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v46, 8, v145
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v61, 8, v131
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v72, 8, v130
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[100:101], 24, v[144:145]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[101:102], 24, v[37:38]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[102:103], 24, v[35:36]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v57, 8, v144
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v75, 24, v115
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v77, 8, v115
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v79, 8, v114
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v93, 8, v99
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v95, 8, v98
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v105, 24, v83
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v107, 8, v83
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v110, 8, v82
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v125, 8, v68
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v140, 8, v66
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v142, 8, v65
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v143, 8, v64
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v40, 8, v54
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v45, 8, v52
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v60, 8, v50
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v76, 8, v48
; GFX11-TRUE16-NEXT: .LBB90_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v131.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v111.l
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v70.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, 0
-; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v129.h
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v1.l, v2.l
-; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v128.h
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v1.h, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v106.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v105.l
-; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v69.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v6, v1
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v3.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.h
-; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v146.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v104.l
-; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v3.h, v4.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v95.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v6, v2
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v135.h
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v3.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.h
-; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v133.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v93.l
-; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v68.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v6, v3
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v5.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v166.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v88.l
-; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v5.h, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v6, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v78.l
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v7.l
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v150.h
-; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v151.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v76.l
-; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v11.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v66, v6, v10
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v43.h
-; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v7.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v74.l
-; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v67.l
-; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v62.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v67, v6, v8
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v177.h
-; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v7.l, v7.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v63.l
-; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v178.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v60.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v180.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v68, v6, v8
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v73.h
-; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v7.l, v7.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v57.l
-; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v9.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v64.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v53.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v69, v6, v8
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v41.h
-; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v7.l, v7.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v47.l
-; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v44.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v45.l
-; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v9.l, v9.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v6, v7
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v92.h
-; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v8.l, v8.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v40.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v10.l, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v6, v8
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v59.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v182.l
-; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v13.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v50.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v6, v9
-; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v89.h
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v108.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v176.l
-; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v11.l, v11.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v6, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v163.l
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v91.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v165.l
-; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v15.h
-; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v12.l, v12.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v6, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v110.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v161.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v38.l
-; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v109.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v6, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v147.l
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v13.l, v13.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v6.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v107.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v149.l
-; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v14.l, v14.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v6, v13
-; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v17.h
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v82.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v94.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v65.l
-; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v81.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v6, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v79.l
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v15.l, v15.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v6.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v80.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v90.l
-; GFX11-TRUE16-NEXT: v_or_b16 v16.h, v16.l, v16.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v6, v15
-; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v19.h
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v85.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v77.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v54.l
-; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v84.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v6, v16
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v72.l
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_or_b16 v17.h, v17.l, v17.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v6.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v83.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v75.l
-; GFX11-TRUE16-NEXT: v_or_b16 v18.h, v18.l, v18.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v6, v17
-; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v21.h
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v96.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v61.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v51.l
-; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v87.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v6, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v56.l
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v19.l, v19.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v6.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v86.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v58.l
-; GFX11-TRUE16-NEXT: v_or_b16 v20.h, v20.l, v20.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v6, v19
-; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v23.h
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v99.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v46.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v48.l
-; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v98.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v6, v20
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v183.l
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_or_b16 v21.h, v21.l, v21.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v6.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v97.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v42.l
-; GFX11-TRUE16-NEXT: v_or_b16 v22.h, v22.l, v22.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v6, v21
-; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v25.h
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v102.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v181.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v36.l
-; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v101.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v6, v22
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v167.l
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_or_b16 v23.h, v23.l, v23.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v100.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v179.l
-; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v24.l, v24.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v6, v23
-; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v27.h
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v113.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v164.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 8, v35.l
-; GFX11-TRUE16-NEXT: v_and_b16 v26.l, 0xff, v112.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v6, v24
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.h, 8, v160.l
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v25.l, v25.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v6.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v103.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v162.l
-; GFX11-TRUE16-NEXT: v_or_b16 v26.h, v26.l, v26.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v6, v25
-; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v29.h
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v116.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v148.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.h, 8, v34.l
-; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v115.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v6, v26
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v144.l
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_or_b16 v27.h, v27.l, v27.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v6.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v114.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v145.l
-; GFX11-TRUE16-NEXT: v_or_b16 v28.h, v28.l, v28.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v6, v27
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v31.h
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v119.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v134.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.h, 8, v33.l
-; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v118.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v6, v28
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v130.l
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_or_b16 v29.h, v29.l, v29.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v6.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v117.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v132.l
+; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v180.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v143.l
+; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v64.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v146.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
+; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v65.h
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v1.l, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v141.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.l, v2.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v179.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v142.l
+; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v66.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v134.l
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v5, v1
+; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v67.h
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v2.l, v2.h
+; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v3.l, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v43.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v140.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v127.l
+; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v68.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v133.l
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v3.l, v3.h
+; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v4.l, v4.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v183.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v139.l
+; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v69.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v121.l
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v82.h
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v4.l, v4.h
+; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v6.l, v6.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v62.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v125.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v132.l
+; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v83.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v5, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v105.l
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v6.l, v6.h
+; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.l, v7.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v47.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v123.l
+; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v98.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v128.l
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v5, v6
+; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v99.h
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v7.l, v7.h
+; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v8.l, v8.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v91.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v110.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v90.l
+; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v114.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v5, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v116.l
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v8.l, v8.h
+; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v9.l, v9.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v74.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v107.l
+; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v115.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v75.l
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v5, v8
+; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v130.h
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v9.l, v9.h
+; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v10.l, v10.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v111.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v95.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v112.l
+; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v131.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v5, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v59.l
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v10.l, v10.h
+; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v11.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v89.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v93.l
+; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v144.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v100.l
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v5, v10
+; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v145.h
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v11.l, v11.h
+; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v12.l, v12.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v138.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v79.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v44.l
+; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v31.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v5, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v118.l
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v12.l, v12.h
+; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v13.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v108.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v77.l
+; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v32.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v124.l
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v5, v12
+; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v33.h
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v13.l, v13.h
+; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v14.l, v14.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v153.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v72.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v117.l
+; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v34.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v5, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v109.l
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v14.l, v14.h
+; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.l, v15.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v137.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v61.l
+; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v35.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v102.l
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v5, v14
+; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v36.h
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v15.l, v15.h
+; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v16.l, v16.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v154.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v57.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v94.l
+; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v37.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v5, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 8, v101.l
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v16.l, v16.h
+; GFX11-TRUE16-NEXT: v_or_b16 v16.h, v17.l, v17.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v152.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v46.l
+; GFX11-TRUE16-NEXT: v_and_b16 v26.l, 0xff, v38.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.h, 8, v78.l
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v5, v16
+; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v48.h
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v17.l, v17.h
+; GFX11-TRUE16-NEXT: v_or_b16 v17.h, v18.l, v18.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v18.l, 0xff, v148.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 8, v136.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.h, 8, v96.l
+; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v49.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, v5, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v63.l
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v18.l, v18.h
+; GFX11-TRUE16-NEXT: v_or_b16 v18.h, v19.l, v19.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v19.l, 0xff, v39.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v126.l
+; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v50.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.h, 8, v86.l
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, v5, v18
+; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v51.h
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v19.l, v19.h
+; GFX11-TRUE16-NEXT: v_or_b16 v19.h, v20.l, v20.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v20.l, 0xff, v150.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 8, v122.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v56.l
+; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v52.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, v5, v19
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v31.h, 8, v80.l
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v20.l, v20.h
+; GFX11-TRUE16-NEXT: v_or_b16 v20.h, v21.l, v21.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v21.l, 0xff, v149.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.h, 8, v120.l
+; GFX11-TRUE16-NEXT: v_and_b16 v32.l, 0xff, v53.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.h, 8, v41.l
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, v5, v20
+; GFX11-TRUE16-NEXT: v_and_b16 v33.l, 0xff, v54.h
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v21.l, v21.h
+; GFX11-TRUE16-NEXT: v_or_b16 v21.h, v22.l, v22.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v22.l, 0xff, v160.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.h, 8, v106.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v70.l
+; GFX11-TRUE16-NEXT: v_and_b16 v34.l, 0xff, v55.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, v5, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v34.h, 8, v181.l
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v22.l, v22.h
+; GFX11-TRUE16-NEXT: v_or_b16 v22.h, v23.l, v23.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v23.l, 0xff, v151.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v23.h, 8, v104.l
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off
-; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[66:69], off offset:16
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v6, v29
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v30.l, v30.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.h
+; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[6:9], off offset:16
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, v5, v22
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v23.l, v23.h
+; GFX11-TRUE16-NEXT: v_or_b16 v23.h, v24.l, v24.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v24.l, 0xff, v162.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v24.h, 8, v92.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, v5, v23
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v24.l, v24.h
+; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v25.l, v25.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v25.l, 0xff, v161.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 8, v88.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, v5, v24
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v25.l, v25.h
+; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v26.l, v26.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v26.l, 0xff, v164.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.h, 8, v76.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, v5, v25
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v26.l, v26.h
+; GFX11-TRUE16-NEXT: v_or_b16 v26.h, v27.l, v27.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v27.l, 0xff, v163.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v27.h, 8, v73.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, v5, v26
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v27.l, v27.h
+; GFX11-TRUE16-NEXT: v_or_b16 v27.h, v28.l, v28.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v28.l, 0xff, v166.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 8, v60.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, v5, v27
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v28.l, v28.h
+; GFX11-TRUE16-NEXT: v_or_b16 v28.h, v29.l, v29.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v29.l, 0xff, v165.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.h, 8, v58.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, v5, v28
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v29.l, v29.h
+; GFX11-TRUE16-NEXT: v_or_b16 v29.h, v30.l, v30.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v30.l, 0xff, v176.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v30.h, 8, v45.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, v5, v29
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v30.l, v30.h
+; GFX11-TRUE16-NEXT: v_or_b16 v30.h, v31.l, v31.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v31.l, 0xff, v167.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v31.h, 8, v42.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v5, v30
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v31.l, v31.h
+; GFX11-TRUE16-NEXT: v_or_b16 v31.h, v32.l, v32.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v32.l, 0xff, v178.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v32.h, 8, v40.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, v5, v31
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v32.l, v32.h
+; GFX11-TRUE16-NEXT: v_or_b16 v32.h, v33.l, v33.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v33.l, 0xff, v177.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v33.h, 8, v182.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v32, v5, v32
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v33.l, v33.h
+; GFX11-TRUE16-NEXT: v_or_b16 v33.h, v34.l, v34.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v5.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, v6, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v33, v5, v33
; GFX11-TRUE16-NEXT: s_clause 0x5
-; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[7:10], off offset:32
-; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[11:14], off offset:48
-; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[15:18], off offset:64
-; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[19:22], off offset:80
-; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[23:26], off offset:96
-; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[27:30], off offset:112
+; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[10:13], off offset:32
+; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[14:17], off offset:48
+; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[18:21], off offset:64
+; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[22:25], off offset:80
+; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[26:29], off offset:96
+; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[30:33], off offset:112
; GFX11-TRUE16-NEXT: s_clause 0x1f
-; GFX11-TRUE16-NEXT: scratch_load_b32 v111, off, s32 offset:12
-; GFX11-TRUE16-NEXT: scratch_load_b32 v110, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_b32 v109, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v108, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_b32 v107, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_b32 v106, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_b32 v105, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_b32 v104, off, s32 offset:40
-; GFX11-TRUE16-NEXT: scratch_load_b32 v95, off, s32 offset:44
-; GFX11-TRUE16-NEXT: scratch_load_b32 v94, off, s32 offset:48
-; GFX11-TRUE16-NEXT: scratch_load_b32 v93, off, s32 offset:52
-; GFX11-TRUE16-NEXT: scratch_load_b32 v92, off, s32 offset:56
-; GFX11-TRUE16-NEXT: scratch_load_b32 v91, off, s32 offset:60
-; GFX11-TRUE16-NEXT: scratch_load_b32 v90, off, s32 offset:64
-; GFX11-TRUE16-NEXT: scratch_load_b32 v89, off, s32 offset:68
-; GFX11-TRUE16-NEXT: scratch_load_b32 v88, off, s32 offset:72
-; GFX11-TRUE16-NEXT: scratch_load_b32 v79, off, s32 offset:76
-; GFX11-TRUE16-NEXT: scratch_load_b32 v78, off, s32 offset:80
-; GFX11-TRUE16-NEXT: scratch_load_b32 v77, off, s32 offset:84
-; GFX11-TRUE16-NEXT: scratch_load_b32 v76, off, s32 offset:88
-; GFX11-TRUE16-NEXT: scratch_load_b32 v75, off, s32 offset:92
-; GFX11-TRUE16-NEXT: scratch_load_b32 v74, off, s32 offset:96
-; GFX11-TRUE16-NEXT: scratch_load_b32 v73, off, s32 offset:100
-; GFX11-TRUE16-NEXT: scratch_load_b32 v72, off, s32 offset:104
-; GFX11-TRUE16-NEXT: scratch_load_b32 v63, off, s32 offset:108
-; GFX11-TRUE16-NEXT: scratch_load_b32 v62, off, s32 offset:112
-; GFX11-TRUE16-NEXT: scratch_load_b32 v61, off, s32 offset:116
-; GFX11-TRUE16-NEXT: scratch_load_b32 v60, off, s32 offset:120
-; GFX11-TRUE16-NEXT: scratch_load_b32 v59, off, s32 offset:124
-; GFX11-TRUE16-NEXT: scratch_load_b32 v58, off, s32 offset:128
-; GFX11-TRUE16-NEXT: scratch_load_b32 v57, off, s32 offset:132
-; GFX11-TRUE16-NEXT: scratch_load_b32 v56, off, s32 offset:136
-; GFX11-TRUE16-NEXT: s_clause 0x7
-; GFX11-TRUE16-NEXT: scratch_load_b32 v47, off, s32 offset:140
-; GFX11-TRUE16-NEXT: scratch_load_b32 v46, off, s32 offset:144
-; GFX11-TRUE16-NEXT: scratch_load_b32 v45, off, s32 offset:148
-; GFX11-TRUE16-NEXT: scratch_load_b32 v44, off, s32 offset:152
-; GFX11-TRUE16-NEXT: scratch_load_b32 v43, off, s32 offset:156
-; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s32 offset:160
-; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s32 offset:164
-; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s32 offset:168
+; GFX11-TRUE16-NEXT: scratch_load_b32 v154, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_b32 v153, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_b32 v152, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_b32 v143, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_b32 v142, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_b32 v141, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_b32 v140, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_b32 v139, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_b32 v138, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_b32 v137, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_b32 v136, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_b32 v127, off, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_load_b32 v126, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_b32 v125, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_b32 v124, off, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_load_b32 v123, off, s32 offset:72
+; GFX11-TRUE16-NEXT: scratch_load_b32 v122, off, s32 offset:76
+; GFX11-TRUE16-NEXT: scratch_load_b32 v121, off, s32 offset:80
+; GFX11-TRUE16-NEXT: scratch_load_b32 v120, off, s32 offset:84
+; GFX11-TRUE16-NEXT: scratch_load_b32 v111, off, s32 offset:88
+; GFX11-TRUE16-NEXT: scratch_load_b32 v110, off, s32 offset:92
+; GFX11-TRUE16-NEXT: scratch_load_b32 v109, off, s32 offset:96
+; GFX11-TRUE16-NEXT: scratch_load_b32 v108, off, s32 offset:100
+; GFX11-TRUE16-NEXT: scratch_load_b32 v107, off, s32 offset:104
+; GFX11-TRUE16-NEXT: scratch_load_b32 v106, off, s32 offset:108
+; GFX11-TRUE16-NEXT: scratch_load_b32 v105, off, s32 offset:112
+; GFX11-TRUE16-NEXT: scratch_load_b32 v104, off, s32 offset:116
+; GFX11-TRUE16-NEXT: scratch_load_b32 v95, off, s32 offset:120
+; GFX11-TRUE16-NEXT: scratch_load_b32 v94, off, s32 offset:124
+; GFX11-TRUE16-NEXT: scratch_load_b32 v93, off, s32 offset:128
+; GFX11-TRUE16-NEXT: scratch_load_b32 v92, off, s32 offset:132
+; GFX11-TRUE16-NEXT: scratch_load_b32 v91, off, s32 offset:136
+; GFX11-TRUE16-NEXT: s_clause 0x1a
+; GFX11-TRUE16-NEXT: scratch_load_b32 v90, off, s32 offset:140
+; GFX11-TRUE16-NEXT: scratch_load_b32 v89, off, s32 offset:144
+; GFX11-TRUE16-NEXT: scratch_load_b32 v88, off, s32 offset:148
+; GFX11-TRUE16-NEXT: scratch_load_b32 v79, off, s32 offset:152
+; GFX11-TRUE16-NEXT: scratch_load_b32 v78, off, s32 offset:156
+; GFX11-TRUE16-NEXT: scratch_load_b32 v77, off, s32 offset:160
+; GFX11-TRUE16-NEXT: scratch_load_b32 v76, off, s32 offset:164
+; GFX11-TRUE16-NEXT: scratch_load_b32 v75, off, s32 offset:168
+; GFX11-TRUE16-NEXT: scratch_load_b32 v74, off, s32 offset:172
+; GFX11-TRUE16-NEXT: scratch_load_b32 v73, off, s32 offset:176
+; GFX11-TRUE16-NEXT: scratch_load_b32 v72, off, s32 offset:180
+; GFX11-TRUE16-NEXT: scratch_load_b32 v63, off, s32 offset:184
+; GFX11-TRUE16-NEXT: scratch_load_b32 v62, off, s32 offset:188
+; GFX11-TRUE16-NEXT: scratch_load_b32 v61, off, s32 offset:192
+; GFX11-TRUE16-NEXT: scratch_load_b32 v60, off, s32 offset:196
+; GFX11-TRUE16-NEXT: scratch_load_b32 v59, off, s32 offset:200
+; GFX11-TRUE16-NEXT: scratch_load_b32 v58, off, s32 offset:204
+; GFX11-TRUE16-NEXT: scratch_load_b32 v57, off, s32 offset:208
+; GFX11-TRUE16-NEXT: scratch_load_b32 v56, off, s32 offset:212
+; GFX11-TRUE16-NEXT: scratch_load_b32 v47, off, s32 offset:216
+; GFX11-TRUE16-NEXT: scratch_load_b32 v46, off, s32 offset:220
+; GFX11-TRUE16-NEXT: scratch_load_b32 v45, off, s32 offset:224
+; GFX11-TRUE16-NEXT: scratch_load_b32 v44, off, s32 offset:228
+; GFX11-TRUE16-NEXT: scratch_load_b32 v43, off, s32 offset:232
+; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s32 offset:236
+; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s32 offset:240
+; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s32 offset:244
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -216125,536 +216365,587 @@ define <64 x half> @bitcast_v64bf16_to_v64f16(<64 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v64bf16_to_v64f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_clause 0xf
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v40, s32 offset:68
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v42, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v43, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v44, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v45, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v46, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v47, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v56, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v57, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v58, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v59, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v60, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v61, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v62, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v63, s32 offset:8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v48, v16
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_b32 v32, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: scratch_load_b32 v16, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_b32 v63, off, s32
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v62, v30 :: v_dual_mov_b32 v61, v29
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v60, v28 :: v_dual_mov_b32 v59, v27
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v58, v26 :: v_dual_mov_b32 v57, v25
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v56, v24 :: v_dual_mov_b32 v55, v23
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v54, v22 :: v_dual_mov_b32 v53, v21
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v52, v20 :: v_dual_mov_b32 v51, v19
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v50, v18 :: v_dual_mov_b32 v49, v17
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_vgpr33_vgpr34_vgpr35_vgpr36_vgpr37_vgpr38_vgpr39_vgpr40_vgpr41_vgpr42_vgpr43_vgpr44_vgpr45_vgpr46_vgpr47
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v32
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB100_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v16
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v20
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v23
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v47, v15 :: v_dual_mov_b32 v46, v14
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, v48 :: v_dual_mov_b32 v17, v49
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v45, v13 :: v_dual_mov_b32 v44, v12
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v43, v11 :: v_dual_mov_b32 v42, v10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v41, v9 :: v_dual_mov_b32 v40, v8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, v7 :: v_dual_mov_b32 v38, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v37, v5 :: v_dual_mov_b32 v36, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, v3 :: v_dual_mov_b32 v34, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, v1 :: v_dual_mov_b32 v32, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, v50 :: v_dual_mov_b32 v19, v51
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v20, v52 :: v_dual_mov_b32 v21, v53
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v22, v54 :: v_dual_mov_b32 v23, v55
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v24, v56 :: v_dual_mov_b32 v25, v57
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, v58 :: v_dual_mov_b32 v27, v59
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v28, v60 :: v_dual_mov_b32 v29, v61
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v30, v62 :: v_dual_mov_b32 v31, v63
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48
+; GFX11-TRUE16-NEXT: .LBB100_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB100_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v49
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v48
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v50
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xffff0000, v54
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xffff0000, v56
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v58
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v60
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v62
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v24, v18, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, 0x400000, v18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v4
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v71, 0x40c00000, v71
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v32
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v8
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v84, 0xffff0000, v9
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v33, 0x40c00000, v33
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v85, 0xffff0000, v10
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v17
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v87, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v33, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v96, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v97, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v21
-; GFX11-TRUE16-NEXT: v_add3_u32 v37, v37, v33, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v18
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v34, 0x40c00000, v16
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v32, 16, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v35, 0x40c00000, v35
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v34, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v16, v16, v32, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v34
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v32, 0x400000, v33
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v24
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v34, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v36, 16, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v26
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v17, v17, v39 :: v_dual_lshlrev_b32 v26, 16, v26
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v34, v36, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v34, 0x400000, v36
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v29
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v32, v37, v32 :: v_dual_lshlrev_b32 v29, 16, v29
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v35, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v24, v24, v18, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v48
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v49
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v84, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_lshlrev_b32 v27, 16, v53
+; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v16, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v16
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_lshlrev_b32 v29, 16, v55
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v21, v21, v16, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v57
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v59
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v16, v21, v22 :: v_dual_lshlrev_b32 v35, 16, v61
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v17, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v17
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v51
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v31
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v31, 16, v31
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v38, 0x40c00000, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v33, v34, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v37, v35, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v34, 0x400000, v35
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v37, 0x40c00000, v39
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v38, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v35, 0x400000, v38
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v80, 0x40c00000, v80
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v81, 0x40c00000, v81 :: v_dual_lshlrev_b32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v33, v34, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v36, v38, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v37, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v38, 0x40c00000, v48 :: v_dual_add_f32 v39, 0x40c00000, v19
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v48, 0x40c00000, v20
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v19, v34, v35
-; GFX11-TRUE16-NEXT: v_add3_u32 v34, v36, v37, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v35, 0x400000, v37
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v39, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v38, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v82, 0x40c00000, v82
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v83, 0x40c00000, v83 :: v_dual_lshlrev_b32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v34, v35, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v36, v39, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v39
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v39, v39
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v39, 0x40c00000, v49
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v49, 0x40c00000, v21
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v35, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v37, v38, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v38
-; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v48, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
-; GFX11-TRUE16-NEXT: v_bfe_u32 v38, v39, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v50, 0x40c00000, v21
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v84, 0x40c00000, v84
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v35, v35, v36, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v36, v37, v48, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v48
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
-; GFX11-TRUE16-NEXT: v_bfe_u32 v48, v49, 16, 1
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v85, 0x40c00000, v85 :: v_dual_lshlrev_b32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v86, 0x40c00000, v86
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v36, v36, v37, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v37, v38, v39, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v39
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v39, v39
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v36.h
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v87, 0x40c00000, v87 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v21, v37, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v37, v48, v49, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v49
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v48, 0x40c00000, v22
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v96, 0x40c00000, v96
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v22, v37, v38 :: v_dual_add_f32 v49, 0x40c00000, v51
-; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v50, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v50
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v50, v50
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v51, 0x40c00000, v23 :: v_dual_add_f32 v14, 0x40c00000, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v63
+; GFX11-TRUE16-NEXT: v_add3_u32 v23, v23, v17, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v48, v23, v25, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v19, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v18, 0x40c00000, v20 :: v_dual_lshlrev_b32 v25, 16, v51
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v50
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v37, v39, v50, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v48, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v50, v49, 16, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v15
-; GFX11-TRUE16-NEXT: v_bfe_u32 v101, v14, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v37, v37, v38, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v38, v39, v48, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v48
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
-; GFX11-TRUE16-NEXT: v_add3_u32 v101, v101, v14, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v112, 0x400000, v14
+; GFX11-TRUE16-NEXT: v_add3_u32 v21, v21, v19, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v24, v26, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v19
+; GFX11-TRUE16-NEXT: v_bfe_u32 v24, v18, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff0000, v52
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v48.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v19, v24, v18, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v49, v21, v23, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v18
+; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v20, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v24, v22, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v49.h
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v19, v21, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v19, v23, v20, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v20
+; GFX11-TRUE16-NEXT: v_add3_u32 v23, v24, v22, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v24, 0x40c00000, v25
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v22
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v24, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v50, v19, v21, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v21, 0x40c00000, v26 :: v_dual_lshlrev_b32 v22, 16, v52
+; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v24, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v19, v23, v25, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v24
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v98, 0x40c00000, v98 :: v_dual_cndmask_b32 v23, v38, v39
-; GFX11-TRUE16-NEXT: v_add3_u32 v38, v50, v49, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v49
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v50, 0x40c00000, v52
-; GFX11-TRUE16-NEXT: v_bfe_u32 v48, v51, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX11-TRUE16-NEXT: v_bfe_u32 v102, v98, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v114, 0x400000, v98
-; GFX11-TRUE16-NEXT: v_bfe_u32 v49, v50, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v38, v38, v39, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v39, v48, v51, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v48, 0x400000, v51
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v51, v51
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v25
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v52, 0x40c00000, v24 :: v_dual_lshlrev_b32 v25, 16, v25
-; GFX11-TRUE16-NEXT: v_add3_u32 v102, v102, v98, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v24, v39, v48, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v39, v49, v50, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v48, 0x400000, v50
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v50, v50
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v51, 0x40c00000, v53
-; GFX11-TRUE16-NEXT: v_bfe_u32 v49, v52, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v26, v21, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v22, 0x40c00000, v22
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v21
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v51, v20, v25, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v20, v26, v21, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v53
+; GFX11-TRUE16-NEXT: v_bfe_u32 v25, v22, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v51.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v20, v20, v24 :: v_dual_add_f32 v23, 0x40c00000, v23
+; GFX11-TRUE16-NEXT: v_add3_u32 v21, v25, v22, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v22
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v39, v39, v48, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v50, v51, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v26, v23, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v52, v21, v24, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v25, v26, v23, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v26, 0x40c00000, v27
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, 0x400000, v23
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v23, 0x40c00000, v28 :: v_dual_lshlrev_b32 v24, 16, v54
+; GFX11-TRUE16-NEXT: v_bfe_u32 v22, v26, 16, 1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v48, v49, v52, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v49, 0x400000, v52
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v52, 0x40c00000, v54 :: v_dual_add_f32 v53, 0x40c00000, v25
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v54, 0x40c00000, v26
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v25, v48, v49 :: v_dual_and_b32 v26, 0xffff0000, v27
-; GFX11-TRUE16-NEXT: v_add3_u32 v48, v50, v51, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v49, 0x400000, v51
-; GFX11-TRUE16-NEXT: v_bfe_u32 v50, v53, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v51, v51
-; GFX11-TRUE16-NEXT: v_bfe_u32 v51, v52, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v21, v25, v27, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v55
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, 0x400000, v26
+; GFX11-TRUE16-NEXT: v_bfe_u32 v28, v23, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v22, v22, v26, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v24, 0x40c00000, v24 :: v_dual_add_f32 v25, 0x40c00000, v25
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, 0x400000, v23
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v50.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v53, v22, v27, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v22, v28, v23, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v27, v24, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v28, v25, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v53.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v48, v48, v49, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v49, v50, v53, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v50, 0x400000, v53
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX11-TRUE16-NEXT: v_bfe_u32 v53, v54, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v49, v49, v50, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v50, v51, v52, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v51, 0x400000, v52
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v55, 0x40c00000, v26
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v49.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v23, v27, v24, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v27, v28, v25, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v22, v22, v26, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, 0x400000, v24
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v28, 0x40c00000, v29
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, 0x400000, v25
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v24, v28, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v54, v23, v26, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v25, 0x40c00000, v30 :: v_dual_lshlrev_b32 v26, 16, v56
+; GFX11-TRUE16-NEXT: v_add3_u32 v24, v24, v28, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v52.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v23, v27, v29, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff0000, v57
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, 0x400000, v28
+; GFX11-TRUE16-NEXT: v_bfe_u32 v30, v25, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v26, 0x40c00000, v26
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v27, 0x40c00000, v27
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, 0x400000, v25
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v55, v24, v29, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v24, v30, v25, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v29, v26, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-TRUE16-NEXT: v_bfe_u32 v30, v27, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v55.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v26, v50, v51, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v50, v53, v54, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v51, 0x400000, v54
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v28
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX11-TRUE16-NEXT: v_bfe_u32 v52, v55, 16, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v54, 0x40c00000, v64 :: v_dual_add_f32 v53, 0x40c00000, v27
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v27, v50, v51, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v50, v52, v55, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v51, 0x400000, v55
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX11-TRUE16-NEXT: v_bfe_u32 v52, v53, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v55, v54, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v64, 0x40c00000, v28
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v50, v50, v51, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v51, v52, v53, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v52, 0x400000, v53
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v28, v51, v52, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v51, v55, v54, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v52, 0x400000, v54
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v55, 0x40c00000, v65
-; GFX11-TRUE16-NEXT: v_bfe_u32 v53, v64, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v54, v55, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v51, v51, v52, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v52, v53, v64, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v53, 0x400000, v64
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v30
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v65, 0x40c00000, v29 :: v_dual_lshlrev_b32 v30, 16, v30
+; GFX11-TRUE16-NEXT: v_add3_u32 v25, v29, v26, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v24, v24, v28, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v28, 0x400000, v26
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX11-TRUE16-NEXT: v_add3_u32 v29, v30, v27, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v30, 0x40c00000, v31
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, 0x400000, v27
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v64, v25, v28, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v29, v52, v53, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v52, v54, v55, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v53, 0x400000, v55
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v64, 0x40c00000, v66
-; GFX11-TRUE16-NEXT: v_bfe_u32 v54, v65, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v52, v52, v53, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v55, v64, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v53, v54, v65, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v54, 0x400000, v65
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v65, 0x40c00000, v67 :: v_dual_add_f32 v66, 0x40c00000, v30
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v67, 0x40c00000, v31
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v30, v53, v54 :: v_dual_and_b32 v31, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v53, v55, v64, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v54, 0x400000, v64
-; GFX11-TRUE16-NEXT: v_bfe_u32 v55, v66, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX11-TRUE16-NEXT: v_bfe_u32 v64, v65, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v26, v30, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v27, 0x40c00000, v32 :: v_dual_lshlrev_b32 v28, 16, v58
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v54.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v25, v29, v31, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff0000, v59
+; GFX11-TRUE16-NEXT: v_add3_u32 v26, v26, v30, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, 0x400000, v30
+; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v27, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v28, 0x40c00000, v28 :: v_dual_add_f32 v29, 0x40c00000, v29
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, 0x400000, v27
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v65, v26, v31, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v26, v32, v27, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v31, v28, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v29, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v65.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v53, v53, v54, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v54, v55, v66, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v55, 0x400000, v66
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v66, v66
-; GFX11-TRUE16-NEXT: v_bfe_u32 v66, v67, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v54, v54, v55, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v55, v64, v65, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v64, 0x400000, v65
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v65, v65
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v68, 0x40c00000, v31
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v54.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v27, v31, v28, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v31, v32, v29, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v26, v26, v30, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, 0x400000, v28
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v32, 0x40c00000, v33
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v29
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v28, v32, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v66, v27, v30, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v29, 0x40c00000, v34 :: v_dual_lshlrev_b32 v30, 16, v60
+; GFX11-TRUE16-NEXT: v_add3_u32 v28, v28, v32, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v64.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v27, v31, v33, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff0000, v61
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v32
+; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v29, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v31, 0x40c00000, v31
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v32, 0x400000, v29
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v67, v28, v33, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v28, v34, v29, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v30, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v31, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v67.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v31, v55, v64, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v55, v66, v67, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v64, 0x400000, v67
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v67, v67
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v65, v68, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v29, v33, v30, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v28, v28, v32, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v32, 0x400000, v30
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
+; GFX11-TRUE16-NEXT: v_add3_u32 v33, v34, v31, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v34, 0x40c00000, v35
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v35, 0x400000, v31
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v68, v29, v32, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v31, v31
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v30, v34, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v31, 0x40c00000, v36 :: v_dual_lshlrev_b32 v32, 16, v62
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v66.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v29, v33, v35, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v63
+; GFX11-TRUE16-NEXT: v_add3_u32 v30, v30, v34, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v35, 0x400000, v34
+; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v31, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v32, 0x40c00000, v32 :: v_dual_add_f32 v33, 0x40c00000, v33
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v34, 0x400000, v31
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v69, v30, v35, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v30, v36, v31, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v32, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v33, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v31, v31
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v69.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v31, v35, v32, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v35, v36, v33, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v30, v30, v34, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v34, 0x400000, v32
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v37
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v33
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v36, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v70, v31, v34, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v33, 0x40c00000, v38
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v32, v32, v36, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v38, 0x40c00000, v0 :: v_dual_cndmask_b32 v31, v35, v37
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v35, 0x400000, v36
+; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v33, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v34, 0x40c00000, v34
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
+; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v38, 16, 1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v67, 0x40c00000, v69 :: v_dual_add_f32 v66, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v55, v64, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v55, v65, v68, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v64, 0x400000, v68
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v68, v68
-; GFX11-TRUE16-NEXT: v_bfe_u32 v65, v66, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v68, v67, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v69, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v55, v55, v64, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v64, v65, v66, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v65, 0x400000, v66
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v66, v66
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v66, 0x400000, v67
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v70, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v64, v64, v65, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v65, v68, v67, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v68, v69, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v67, v67
-; GFX11-TRUE16-NEXT: v_bfe_u32 v67, v70, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v64.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v65, v66, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v65, v68, v69, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v66, 0x400000, v69
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v69, v69
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v69, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v68.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v34
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v32, v35, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v32, v37, v33, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v35, 0x400000, v33
+; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v34, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
+; GFX11-TRUE16-NEXT: v_add3_u32 v33, v36, v38, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v36, v37, v34, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v32, v35, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v35, 0x400000, v38
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v37, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v37
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v68, 0x40c00000, v68 :: v_dual_cndmask_b32 v65, v65, v66
-; GFX11-TRUE16-NEXT: v_add3_u32 v66, v67, v70, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v67, 0x400000, v70
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v70, v70
-; GFX11-TRUE16-NEXT: v_bfe_u32 v70, v68, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v65.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v33, v35, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v37, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v34, 0x40c00000, v71
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v1.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v66, v66, v67, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v67, v69, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v69, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v37, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v36, v39, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v36, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v34, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v34
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v36, 0x40c00000, v36 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v70.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v71, v35, v38, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v35, v39, v34, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v38, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
+; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v36, 16, 1
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v65, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v28.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v67, v69, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v67, v70, v68, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v69, 0x400000, v68
-; GFX11-TRUE16-NEXT: v_bfe_u32 v70, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v68, v68
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v68, v70, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v67, v67, v69, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v69, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v70, v71, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v71.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v35, v37, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v35, v38, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v38, v39, v36, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v35, v37, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v36, 0x40c00000, v80
+; GFX11-TRUE16-NEXT: v_add3_u32 v37, v37, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v35, v38, v39 :: v_dual_and_b32 v38, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v80, v36, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v66
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v27.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v27, 0xffff, v65, v50
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v25.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v68, v69, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v68, v70, v71, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v69, 0x400000, v71
-; GFX11-TRUE16-NEXT: v_bfe_u32 v70, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v71, v71
-; GFX11-TRUE16-NEXT: v_bfe_u32 v71, v80, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v25, 0xffff, v49, v48
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v17.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v68, v68, v69, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v69, v70, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v70, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v38, 0x40c00000, v38 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v37, v39, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v37, v80, v36, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v36
+; GFX11-TRUE16-NEXT: v_bfe_u32 v80, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
+; GFX11-TRUE16-NEXT: v_bfe_u32 v81, v38, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v3.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v36, v37, v39, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v37, v80, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v67
-; GFX11-TRUE16-NEXT: v_bfi_b32 v26, 0xffff, v66, v26
-; GFX11-TRUE16-NEXT: v_bfi_b32 v16, 0xffff, v48, v16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v69, v70 :: v_dual_add_f32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v69, v71, v80, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v70, 0x400000, v80
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v80, v80
-; GFX11-TRUE16-NEXT: v_bfe_u32 v80, v81, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v71, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v69, v69, v70, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v80, v81, v38, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v81, 0x400000, v38
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v37, v39, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v39, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v38, 0x40c00000, v82
+; GFX11-TRUE16-NEXT: v_add3_u32 v39, v39, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v37, v80, v81 :: v_dual_and_b32 v80, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v81, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v82, v38, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v70, v71, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v71, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v68
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v0.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v64, v55
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v30.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v70, v71, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v70, v80, v81, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v71, 0x400000, v81
-; GFX11-TRUE16-NEXT: v_bfe_u32 v80, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v81, v81
-; GFX11-TRUE16-NEXT: v_bfe_u32 v81, v82, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v29.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v29, 0xffff, v55, v52
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v23.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v70, v70, v71, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v71, v80, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v80, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v80, 0x40c00000, v80 :: v_dual_lshlrev_b32 v7, 16, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v39, v81, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v39, v82, v38, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v81, 0x400000, v38
+; GFX11-TRUE16-NEXT: v_bfe_u32 v82, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
+; GFX11-TRUE16-NEXT: v_bfe_u32 v83, v80, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v5.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v38, v39, v81, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v39, v82, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v81, 0x400000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: v_bfi_b32 v28, 0xffff, v64, v51
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v24.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v6, v71, v80 :: v_dual_add_f32 v7, 0x40c00000, v7
-; GFX11-TRUE16-NEXT: v_add3_u32 v71, v81, v82, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v80, 0x400000, v82
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v82, v82
-; GFX11-TRUE16-NEXT: v_bfe_u32 v82, v83, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v82, v83, v80, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v83, 0x400000, v80
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v39, v81, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v80, v80
; GFX11-TRUE16-NEXT: v_bfe_u32 v81, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v30, 0xffff, v54, v53
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v71, v71, v80, vcc_lo
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v80, 0x40c00000, v84 :: v_dual_cndmask_b32 v39, v82, v83
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v81, v81, v7, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v83, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v84, v80, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_add3_u32 v80, v81, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v81, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v22.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v24, 0xffff, v50, v39
-; GFX11-TRUE16-NEXT: v_bfi_b32 v23, 0xffff, v51, v38
-; GFX11-TRUE16-NEXT: v_bfi_b32 v22, 0xffff, v52, v37
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v80, v81, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v80, v82, v83, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v81, 0x400000, v83
-; GFX11-TRUE16-NEXT: v_bfe_u32 v82, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v83, v83
-; GFX11-TRUE16-NEXT: v_bfe_u32 v83, v84, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v7.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v20.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v19.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v80, v80, v81, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v81, v82, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v82, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v82, 0x40c00000, v82 :: v_dual_lshlrev_b32 v9, 16, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v81, v83, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v81, v84, v80, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v83, 0x400000, v80
+; GFX11-TRUE16-NEXT: v_bfe_u32 v84, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v85, v82, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v80, v80
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v7.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v80, v84, v8, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v84, 0x400000, v82
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v40, v81, v83, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v81, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_add3_u32 v83, v85, v82, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v85, 0xffff0000, v10
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v18.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v7, v71
-; GFX11-TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v6, v70
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v8, v81, v82 :: v_dual_add_f32 v9, 0x40c00000, v9
-; GFX11-TRUE16-NEXT: v_add3_u32 v81, v83, v84, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v82, 0x400000, v84
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v84, v84
-; GFX11-TRUE16-NEXT: v_bfe_u32 v84, v85, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v83, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v5, v69
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v81, v81, v82, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_add3_u32 v82, v83, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v2.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v80, v81, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v80, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v82, v82
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v81, 0x40c00000, v85
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; GFX11-TRUE16-NEXT: v_add3_u32 v80, v80, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v41, v83, v84, vcc_lo
; GFX11-TRUE16-NEXT: v_or_b32_e32 v83, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_bfi_b32 v8, 0xffff, v8, v80
-; GFX11-TRUE16-NEXT: v_bfi_b32 v31, 0xffff, v68, v31
-; GFX11-TRUE16-NEXT: v_bfi_b32 v21, 0xffff, v53, v21
-; GFX11-TRUE16-NEXT: v_bfi_b32 v20, 0xffff, v36, v35
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v82, v83, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v82, v84, v85, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v83, 0x400000, v85
+; GFX11-TRUE16-NEXT: v_bfe_u32 v84, v81, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v82, 0x40c00000, v82
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v40.l, v8.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v85, v82, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v80, v83, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v80, v84, v81, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v83, 0x400000, v81
; GFX11-TRUE16-NEXT: v_bfe_u32 v84, v10, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v85, v85
-; GFX11-TRUE16-NEXT: v_bfe_u32 v85, v86, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v9.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v19, 0xffff, v37, v34
-; GFX11-TRUE16-NEXT: v_bfi_b32 v18, 0xffff, v38, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v82, v82, v83, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v83, v84, v10, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v84, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v81, v81
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v81, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v41.l, v9.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v42, v80, v83, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v80, v84, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v83, v85, v82, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v85, 0xffff0000, v12
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT: v_bfi_b32 v9, 0xffff, v9, v81
-; GFX11-TRUE16-NEXT: v_bfi_b32 v17, 0xffff, v39, v32
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v10, v83, v84 :: v_dual_add_f32 v11, 0x40c00000, v11
-; GFX11-TRUE16-NEXT: v_add3_u32 v83, v85, v86, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v84, 0x400000, v86
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v86, v86
-; GFX11-TRUE16-NEXT: v_bfe_u32 v86, v87, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v85, v11, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v10.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v83, v83, v84, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v84, 0x400000, v82
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v80, v81, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v80, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v82, v82
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v81, 0x40c00000, v85
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; GFX11-TRUE16-NEXT: v_add3_u32 v80, v80, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v43, v83, v84, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v83, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v84, v81, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v84, v85, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v85, 0x400000, v11
-; GFX11-TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v10, v82
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v84, v85, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v84, v86, v87, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v85, 0x400000, v87
-; GFX11-TRUE16-NEXT: v_bfe_u32 v86, v12, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v87, v87
-; GFX11-TRUE16-NEXT: v_bfe_u32 v87, v96, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v11.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v84, v84, v85, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v85, v86, v12, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v86, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v82, 0x40c00000, v82 :: v_dual_lshlrev_b32 v13, 16, v13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v42.l, v10.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v4.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v80, v83, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v80, v84, v81, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v83, 0x400000, v81
+; GFX11-TRUE16-NEXT: v_bfe_u32 v84, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v81, v81
+; GFX11-TRUE16-NEXT: v_bfe_u32 v85, v82, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v81, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v43.l, v11.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v44, v80, v83, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v80, v84, v12, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT: v_bfi_b32 v11, 0xffff, v11, v83
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v12, v85, v86 :: v_dual_add_f32 v13, 0x40c00000, v13
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_add3_u32 v85, v87, v96, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v12.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v86, v13, 16, 1
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v14, v101, v112 :: v_dual_add_f32 v87, 0x40c00000, v97
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v98, v98
+; GFX11-TRUE16-NEXT: v_add3_u32 v83, v85, v82, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v84, 0x400000, v82
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v85, 0xffff0000, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v80, v81, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v80, v13, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v82, v82
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v81, 0x40c00000, v85 :: v_dual_lshlrev_b32 v82, 16, v15
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v80, v80, v13, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v45, v83, v84, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v83, 0x400000, v13
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-TRUE16-NEXT: v_bfe_u32 v84, v81, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v82, 0x40c00000, v82
; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; GFX11-TRUE16-NEXT: v_add3_u32 v86, v86, v13, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v100, 0x400000, v13
-; GFX11-TRUE16-NEXT: v_bfe_u32 v99, v87, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v98, v102, v114 :: v_dual_add_f32 v15, 0x40c00000, v15
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v103, 0x400000, v87
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v87, v87
-; GFX11-TRUE16-NEXT: v_add3_u32 v99, v99, v87, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v97, 0x400000, v96
-; GFX11-TRUE16-NEXT: v_bfe_u32 v113, v15, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v112, 0x400000, v15
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v98.l, v98.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v87, v99, v103, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v86, 0x400000, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v80, v83, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v80, v14, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v83, v84, v81, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v84, v82, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
+; GFX11-TRUE16-NEXT: v_add3_u32 v80, v80, v14, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v96, 0x400000, v82
+; GFX11-TRUE16-NEXT: v_add3_u32 v84, v84, v82, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v85, 0x400000, v81
+; GFX11-TRUE16-NEXT: v_bfe_u32 v87, v15, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v80, v86, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v82, v82
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v86, 0x400000, v15
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v45.l, v13.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v80, v87, v15, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v44.l, v12.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v82, v84, v96, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v81, v81
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v6.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v46, v83, v85, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-TRUE16-NEXT: v_add3_u32 v101, v113, v15, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v14.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v12, 0xffff, v12, v84
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v15, v101, v112, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_bfi_b32 v14, 0xffff, v14, v87
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v15, 0xffff, v98, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v86, v100, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v96, v96
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v13.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v85, v85, v97, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v13, 0xffff, v13, v85
-; GFX11-TRUE16-NEXT: .LBB100_2: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v46.l, v14.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v47, v80, v86, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v47.l, v82.h
+; GFX11-TRUE16-NEXT: .LBB100_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v40 :: v_dual_mov_b32 v9, v41
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, v42 :: v_dual_mov_b32 v11, v43
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v44 :: v_dual_mov_b32 v13, v45
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, v46 :: v_dual_mov_b32 v15, v47
+; GFX11-TRUE16-NEXT: s_clause 0xf
+; GFX11-TRUE16-NEXT: scratch_load_b32 v63, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_b32 v62, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_b32 v61, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_b32 v60, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_b32 v59, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_b32 v58, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_b32 v57, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_b32 v56, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_b32 v47, off, s32 offset:40
+; GFX11-TRUE16-NEXT: scratch_load_b32 v46, off, s32 offset:44
+; GFX11-TRUE16-NEXT: scratch_load_b32 v45, off, s32 offset:48
+; GFX11-TRUE16-NEXT: scratch_load_b32 v44, off, s32 offset:52
+; GFX11-TRUE16-NEXT: scratch_load_b32 v43, off, s32 offset:56
+; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s32 offset:60
+; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s32 offset:64
+; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s32 offset:68
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v32 :: v_dual_mov_b32 v1, v33
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v34 :: v_dual_mov_b32 v3, v35
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v36 :: v_dual_mov_b32 v5, v37
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v38 :: v_dual_mov_b32 v7, v39
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
index 21ec3ee1996a6..9b28fd9e7b6fd 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
@@ -2668,85 +2668,92 @@ define <4 x i32> @bitcast_v8bf16_to_v4i32(<8 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v8bf16_to_v4i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v4
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, v3 :: v_dual_mov_b32 v6, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v5 :: v_dual_add_f32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v5
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v6, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v5
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v6, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v3, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v13, v11, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v8
; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v2, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v13, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v10, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v10, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.h
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v8
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v8, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v1, v9 :: v_dual_and_b32 v0, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v9, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v13, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v14, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v3, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v6, v2
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v4, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v8, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v5, v0
-; GFX11-TRUE16-NEXT: .LBB22_2: ; %end
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.h
+; GFX11-TRUE16-NEXT: .LBB22_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v6 :: v_dual_mov_b32 v3, v7
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8bf16_to_v4i32:
@@ -7118,85 +7125,92 @@ define <4 x float> @bitcast_v8bf16_to_v4f32(<8 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v8bf16_to_v4f32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v4
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, v3 :: v_dual_mov_b32 v6, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v5 :: v_dual_add_f32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v5
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v6, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v5
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v6, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v3, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v13, v11, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v8
; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v2, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v13, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v10, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v10, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.h
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v8
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v8, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v1, v9 :: v_dual_and_b32 v0, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v9, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v13, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v14, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v3, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v6, v2
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v4, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v8, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v5, v0
-; GFX11-TRUE16-NEXT: .LBB46_2: ; %end
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.h
+; GFX11-TRUE16-NEXT: .LBB46_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v6 :: v_dual_mov_b32 v3, v7
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8bf16_to_v4f32:
@@ -11216,85 +11230,92 @@ define <2 x i64> @bitcast_v8bf16_to_v2i64(<8 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v8bf16_to_v2i64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v4
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, v3 :: v_dual_mov_b32 v6, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB66_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB66_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v5 :: v_dual_add_f32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v5
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v6, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v5
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v6, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v3, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v13, v11, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v8
; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v2, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v13, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v10, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v10, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.h
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v8
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v8, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v1, v9 :: v_dual_and_b32 v0, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v9, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v13, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v14, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v3, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v6, v2
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v4, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v8, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v5, v0
-; GFX11-TRUE16-NEXT: .LBB66_2: ; %end
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.h
+; GFX11-TRUE16-NEXT: .LBB66_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v6 :: v_dual_mov_b32 v3, v7
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8bf16_to_v2i64:
@@ -14900,85 +14921,92 @@ define <2 x double> @bitcast_v8bf16_to_v2f64(<8 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v8bf16_to_v2f64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v4
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, v3 :: v_dual_mov_b32 v6, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v5 :: v_dual_add_f32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v5
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v6, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v5
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v6, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v3, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v13, v11, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v8
; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v2, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v13, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v10, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v10, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.h
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v8
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v8, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v1, v9 :: v_dual_and_b32 v0, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v9, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v13, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v14, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v3, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v6, v2
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v4, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v8, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v5, v0
-; GFX11-TRUE16-NEXT: .LBB82_2: ; %end
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.h
+; GFX11-TRUE16-NEXT: .LBB82_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v6 :: v_dual_mov_b32 v3, v7
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8bf16_to_v2f64:
@@ -21093,24 +21121,30 @@ define <8 x half> @bitcast_v8bf16_to_v8f16(<8 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v8bf16_to_v8f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v4
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, v3 :: v_dual_mov_b32 v6, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB102_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB102_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v1.l
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v2.l
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
@@ -21126,55 +21160,53 @@ define <8 x half> @bitcast_v8bf16_to_v8f16(<8 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v1, v7 :: v_dual_and_b32 v7, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v11, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v11, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v7 :: v_dual_add_f32 v7, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v9, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_cndmask_b32 v2, v2, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v3.l
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v8, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v8, v8, v12 :: v_dual_and_b32 v3, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_add_f32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v3, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v7.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v14, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v13, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v13, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v10, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v5, v3
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v8, v0
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v9, v4
-; GFX11-TRUE16-NEXT: .LBB102_2: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v8.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v10.h
+; GFX11-TRUE16-NEXT: .LBB102_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v6 :: v_dual_mov_b32 v3, v7
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8bf16_to_v8f16:
@@ -23752,140 +23784,139 @@ define <16 x i8> @bitcast_v8bf16_to_v16i8(<8 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v8bf16_to_v16i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v3 :: v_dual_mov_b32 v10, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[16:17], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v10
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[17:18], 24, v[2:3]
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v3.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v11.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[2:3]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[0:1]
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v0.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v1.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v2.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v3.h
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3
; GFX11-TRUE16-NEXT: .LBB108_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v6, v8 :: v_dual_and_b32 v3, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v4, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_add_f32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v9, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v11.l
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v8, v10 :: v_dual_and_b32 v1, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v2, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v13, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v0, v9, v13, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v8, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v13
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v0, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v11, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v4.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v3, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v13, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v13
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v0, v8 :: v_dual_and_b32 v5, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v10.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v8, v13, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v3, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v3, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v8, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v10, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v12.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v11, v15, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v14, v10, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v2, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v9, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v11, 0xffff, v1, v14
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v16, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v13, v7
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v11
-; GFX11-TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v9, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v11, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v6.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[16:17], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[17:18], 24, v[2:3]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 8, v16
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v22, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v12.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v21, v1, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v8.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v22
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v22
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[21:22]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v21
; GFX11-TRUE16-NEXT: .LBB108_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v16.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v19.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v17.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v21.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v18.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v12.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v14.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v22.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8bf16_to_v16i8:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
index 38302a75fe26d..52e125d0d658f 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
@@ -3736,18 +3736,29 @@ define <8 x i32> @bitcast_v16bf16_to_v8i32(<16 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v16bf16_to_v8i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, v7 :: v_dual_mov_b32 v14, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, v5 :: v_dual_mov_b32 v12, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v3 :: v_dual_mov_b32 v10, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_lshlrev_b32 v8, 16, v7
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v8, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v9, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v8
@@ -3755,138 +3766,129 @@ define <8 x i32> @bitcast_v16bf16_to_v8i32(<16 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v8, 0x7fff
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v8, v11, v14 :: v_dual_and_b32 v7, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v7 :: v_dual_lshlrev_b32 v6, 16, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v11, v15, v9, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v7, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v7
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v7, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v7, v12, v13
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v15, v12, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v6, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v8, v7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v8.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v9
; GFX11-TRUE16-NEXT: v_add3_u32 v14, v16, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v14, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v11, v12 :: v_dual_add_f32 v10, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v6, v9
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v10, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v13, v10, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v12, v14, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v12, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v10
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v10, v11, v14 :: v_dual_add_f32 v11, 0x40c00000, v15
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v11, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v11, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v6.h
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v10, v7, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v6, v11, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v5, v10
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v14
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v13, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v11, 16, 1
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v8
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v12, v13 :: v_dual_add_f32 v10, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v10, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v10
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v10, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v15, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v9, v14 :: v_dual_and_b32 v14, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v9
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v10, v10, v12 :: v_dual_add_f32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_cndmask_b32 v12, v6, v10
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v4.h
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v4, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v10, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v3.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v13, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v13, 0x40c00000, v14 :: v_dual_lshlrev_b32 v14, 16, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v11, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_add_f32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v14
-; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v13, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v13
-; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v12, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v12
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v13, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v12, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v14, v17, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v14, v18, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v15, v19, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v12.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v11, v16, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v9, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v16, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v4, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v14, v17, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v12, v0
-; GFX11-TRUE16-NEXT: .LBB22_2: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v2, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.h
+; GFX11-TRUE16-NEXT: .LBB22_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v12 :: v_dual_mov_b32 v5, v13
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v14 :: v_dual_mov_b32 v7, v15
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v16bf16_to_v8i32:
@@ -10729,18 +10731,29 @@ define <8 x float> @bitcast_v16bf16_to_v8f32(<16 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v16bf16_to_v8f32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, v7 :: v_dual_mov_b32 v14, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, v5 :: v_dual_mov_b32 v12, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v3 :: v_dual_mov_b32 v10, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_lshlrev_b32 v8, 16, v7
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v8, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v9, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v8
@@ -10748,138 +10761,129 @@ define <8 x float> @bitcast_v16bf16_to_v8f32(<16 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v8, 0x7fff
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v8, v11, v14 :: v_dual_and_b32 v7, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v7 :: v_dual_lshlrev_b32 v6, 16, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v11, v15, v9, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v7, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v7
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v7, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v7, v12, v13
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v15, v12, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v6, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v8, v7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v8.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v9
; GFX11-TRUE16-NEXT: v_add3_u32 v14, v16, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v14, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v11, v12 :: v_dual_add_f32 v10, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v6, v9
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v10, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v13, v10, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v12, v14, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v12, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v10
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v10, v11, v14 :: v_dual_add_f32 v11, 0x40c00000, v15
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v11, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v11, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v6.h
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v10, v7, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v6, v11, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v5, v10
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v14
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v13, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v11, 16, 1
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v8
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v12, v13 :: v_dual_add_f32 v10, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v10, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v10
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v10, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v15, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v9, v14 :: v_dual_and_b32 v14, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v9
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v10, v10, v12 :: v_dual_add_f32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_cndmask_b32 v12, v6, v10
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v4.h
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v4, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v10, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v3.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v13, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v13, 0x40c00000, v14 :: v_dual_lshlrev_b32 v14, 16, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v11, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_add_f32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v14
-; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v13, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v13
-; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v12, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v12
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v13, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v12, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v14, v17, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v14, v18, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v15, v19, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v12.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v11, v16, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v9, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v16, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v4, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v14, v17, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v12, v0
-; GFX11-TRUE16-NEXT: .LBB46_2: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v2, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.h
+; GFX11-TRUE16-NEXT: .LBB46_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v12 :: v_dual_mov_b32 v5, v13
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v14 :: v_dual_mov_b32 v7, v15
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v16bf16_to_v8f32:
@@ -17286,18 +17290,29 @@ define <4 x i64> @bitcast_v16bf16_to_v4i64(<16 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v16bf16_to_v4i64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, v7 :: v_dual_mov_b32 v14, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, v5 :: v_dual_mov_b32 v12, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v3 :: v_dual_mov_b32 v10, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB66_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB66_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_lshlrev_b32 v8, 16, v7
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v8, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v9, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v8
@@ -17305,138 +17320,129 @@ define <4 x i64> @bitcast_v16bf16_to_v4i64(<16 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v8, 0x7fff
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v8, v11, v14 :: v_dual_and_b32 v7, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v7 :: v_dual_lshlrev_b32 v6, 16, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v11, v15, v9, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v7, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v7
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v7, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v7, v12, v13
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v15, v12, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v6, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v8, v7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v8.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v9
; GFX11-TRUE16-NEXT: v_add3_u32 v14, v16, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v14, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v11, v12 :: v_dual_add_f32 v10, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v6, v9
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v10, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v13, v10, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v12, v14, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v12, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v10
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v10, v11, v14 :: v_dual_add_f32 v11, 0x40c00000, v15
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v11, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v11, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v6.h
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v10, v7, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v6, v11, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v5, v10
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v14
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v13, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v11, 16, 1
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v8
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v12, v13 :: v_dual_add_f32 v10, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v10, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v10
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v10, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v15, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v9, v14 :: v_dual_and_b32 v14, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v9
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v10, v10, v12 :: v_dual_add_f32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_cndmask_b32 v12, v6, v10
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v4.h
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v4, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v10, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v3.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v13, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v13, 0x40c00000, v14 :: v_dual_lshlrev_b32 v14, 16, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v11, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_add_f32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v14
-; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v13, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v13
-; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v12, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v12
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v13, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v12, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v14, v17, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v14, v18, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v15, v19, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v12.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v11, v16, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v9, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v16, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v4, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v14, v17, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v12, v0
-; GFX11-TRUE16-NEXT: .LBB66_2: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v2, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.h
+; GFX11-TRUE16-NEXT: .LBB66_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v12 :: v_dual_mov_b32 v5, v13
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v14 :: v_dual_mov_b32 v7, v15
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v16bf16_to_v4i64:
@@ -23304,18 +23310,29 @@ define <4 x double> @bitcast_v16bf16_to_v4f64(<16 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v16bf16_to_v4f64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, v7 :: v_dual_mov_b32 v14, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, v5 :: v_dual_mov_b32 v12, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v3 :: v_dual_mov_b32 v10, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_lshlrev_b32 v8, 16, v7
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v8, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v9, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v8
@@ -23323,138 +23340,129 @@ define <4 x double> @bitcast_v16bf16_to_v4f64(<16 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v8, 0x7fff
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v8, v11, v14 :: v_dual_and_b32 v7, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v7 :: v_dual_lshlrev_b32 v6, 16, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v11, v15, v9, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v7, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v7
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v7, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v7, v12, v13
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v15, v12, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v6, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v8, v7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v8.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v9
; GFX11-TRUE16-NEXT: v_add3_u32 v14, v16, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v14, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v11, v12 :: v_dual_add_f32 v10, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v6, v9
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v10, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v13, v10, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v12, v14, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v12, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v10
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v10, v11, v14 :: v_dual_add_f32 v11, 0x40c00000, v15
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v11, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v11, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v6.h
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v10, v7, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v6, v11, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v5, v10
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v14
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v13, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v11, 16, 1
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v8
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v12, v13 :: v_dual_add_f32 v10, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v10, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v10
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v10, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v15, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v9, v14 :: v_dual_and_b32 v14, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v9
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v10, v10, v12 :: v_dual_add_f32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_cndmask_b32 v12, v6, v10
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v4.h
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v4, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v10, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v3.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v13, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v13, 0x40c00000, v14 :: v_dual_lshlrev_b32 v14, 16, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v11, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_add_f32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v14
-; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v13, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v13
-; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v12, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v12
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v13, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v12, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v14, v17, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v14, v18, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v15, v19, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v12.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v11, v16, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v9, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v16, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v4, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v14, v17, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v12, v0
-; GFX11-TRUE16-NEXT: .LBB82_2: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v2, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.h
+; GFX11-TRUE16-NEXT: .LBB82_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v12 :: v_dual_mov_b32 v5, v13
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v14 :: v_dual_mov_b32 v7, v15
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v16bf16_to_v4f64:
@@ -33716,156 +33724,158 @@ define <16 x half> @bitcast_v16bf16_to_v16f16(<16 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v16bf16_to_v16f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, v7 :: v_dual_mov_b32 v15, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, v5 :: v_dual_mov_b32 v13, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v11, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB102_3
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
+; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB102_4
+; GFX11-TRUE16-NEXT: .LBB102_2: ; %end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-NEXT: .LBB102_3: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v9 :: v_dual_mov_b32 v1, v10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v11 :: v_dual_mov_b32 v3, v12
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v13 :: v_dual_mov_b32 v5, v14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v15 :: v_dual_mov_b32 v7, v16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB102_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v6
+; GFX11-TRUE16-NEXT: .LBB102_4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v10
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v10
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v15
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v8, v11, v12 :: v_dual_and_b32 v9, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v1, 16, v9
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v9, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v13, v15, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v14, v16 :: v_dual_add_f32 v10, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_cndmask_b32 v0, v5, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v10, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v11, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v10
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v14, v10, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v11, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v14, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v12
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v9, v7, v9
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v11, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v12, 0x40c00000, v12 :: v_dual_add_f32 v11, 0x40c00000, v11
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v12, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v11, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v14, v15, v12, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v14, v15, vcc_lo
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v13, v15, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v15, v16, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v4 :: v_dual_add_f32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v10, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v8, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v9.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v5, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v7, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v11
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v14, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v14
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v12
-; GFX11-TRUE16-NEXT: v_add3_u32 v16, v16, v14, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v12 :: v_dual_lshlrev_b32 v6, 16, v13
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v8, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v13, v18, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v7
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v16, v19, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v16, 0x40c00000, v20
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v17.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v14
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v7 :: v_dual_lshlrev_b32 v14, 16, v14
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v10.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v4, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v12, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v6, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v18, 0x40c00000, v18 :: v_dual_add_f32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v16, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v16
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v13, v19, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v12, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v20, v16, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v18, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v18
-; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v18, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v13, v22, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_add3_u32 v13, v23, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v5, v14
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v20, v24, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v19, v21, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v13, v7, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v14
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v5, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v6, v16
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v13, v22, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v18.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v15, v17, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v13, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v11
-; GFX11-TRUE16-NEXT: .LBB102_2: ; %end
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v13, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v18
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v11.h
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v12, v14 :: v_dual_lshlrev_b32 v12, 16, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v13, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v16
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v13
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v13, v6, v15 :: v_dual_and_b32 v16, 0xffff0000, v16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v15, v18, v7, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v14, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v12, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
+; GFX11-TRUE16-NEXT: v_add3_u32 v18, v18, v14, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v14
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v6, v20, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v16, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v13.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v18, v22, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v20, v21, v16, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v16
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v15, v19, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v12.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v20, v21, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v14.h
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -38284,253 +38294,246 @@ define <32 x i8> @bitcast_v16bf16_to_v32i8(<16 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v16bf16_to_v32i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, v7 :: v_dual_mov_b32 v26, v6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v5 :: v_dual_mov_b32 v18, v4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v3 :: v_dual_mov_b32 v10, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[32:33], 24, v[26:27]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[33:34], 24, v[18:19]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v26
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v10
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[2:3]
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v3.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v11.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v19.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.h, v27.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[36:37], 24, v[6:7]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[37:38], 24, v[4:5]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 24, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[38:39], 24, v[2:3]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[48:49], 24, v[0:1]
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v0.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v1.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.h, v2.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.h, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.h, v4.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.h, v5.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.h, v6.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.h, v7.h
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7
; GFX11-TRUE16-NEXT: .LBB108_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v4 :: v_dual_add_f32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v13, 0x40c00000, v10 :: v_dual_add_f32 v8, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v8, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v17, v10, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v11, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v15, v9, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v27, v17, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v8, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v13, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v26, v11, v16, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v12, v4, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v3 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v7, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v3, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v10.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v9, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v3, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v12, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v1, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v12, v14, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v0, v1 :: v_dual_add_f32 v5, 0x40c00000, v15
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v12.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v10
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v19
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v9, v16, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v11, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v19
-; GFX11-TRUE16-NEXT: v_bfi_b32 v11, 0xffff, v1, v14
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v10
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v12, v13, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 8, v27
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v11.h
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v12, v0, v3 :: v_dual_and_b32 v3, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v14
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v13
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v8, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v12.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[48:49], 24, v[26:27]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 8, v26
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v32, v1, v2 :: v_dual_add_f32 v1, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v8, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v33
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v13, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v15, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v10, 0x40c00000, v13
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v8, v8, v13 :: v_dual_add_f32 v1, 0x40c00000, v15
-; GFX11-TRUE16-NEXT: v_add3_u32 v13, v16, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v18
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v10, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v22, v13, v15, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v8.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v35, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v9 :: v_dual_add_f32 v1, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v14, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v9, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v14, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v16, v17, v10, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v10
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v19, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v27
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v16, v17, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v7
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v20.h
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v10, v13 :: v_dual_add_f32 v10, 0x40c00000, v15
-; GFX11-TRUE16-NEXT: v_add3_u32 v15, v16, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v26
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v7, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v28, v1, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v20.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v13, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v16.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v24, v6, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v33
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v51, v3, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v27
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v23, 0x40c00000, v16 :: v_dual_cndmask_b32 v16, v15, v18
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v10, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v13, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v15, v19, v10, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v23, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v13
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v13, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v26
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v23, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, 0x400000, v23
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v28, v5, v24, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v21
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v28.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 24, v27
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v50, v1, v2, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v24.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v51
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v51
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v25, v18, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v24, v19, v26, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v28.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v25, v18, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfi_b32 v19, 0xffff, v17, v22
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v30, v15, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v16.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v8.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v19
-; GFX11-TRUE16-NEXT: v_bfi_b32 v27, 0xffff, v10, v30
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v24.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v18, 0xffff, v15, v1
-; GFX11-TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v21, v2
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v9, v7
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v27
-; GFX11-TRUE16-NEXT: v_bfi_b32 v26, 0xffff, v13, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[32:33], 24, v[26:27]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[33:34], 24, v[18:19]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[2:3]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v26
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v10
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[36:37], 24, v[50:51]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[37:38], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[38:39], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v50
; GFX11-TRUE16-NEXT: .LBB108_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v11.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v26.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v48.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v10.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v27.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v14.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v32.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v38.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v12.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v14.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v33.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v16.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v18.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v33.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v34.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v37.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v20.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v22.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v35.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v24.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v26.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v50.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v36.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v28.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v30.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v51.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v16bf16_to_v32i8:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
index 8e30ee659a260..632b03ca51b81 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
@@ -1303,13 +1303,18 @@ define i32 @bitcast_v2bf16_to_i32(<2 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v2bf16_to_i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v1
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB14_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB14_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -1322,15 +1327,16 @@ define i32 @bitcast_v2bf16_to_i32(<2 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
-; GFX11-TRUE16-NEXT: .LBB14_2: ; %end
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11-TRUE16-NEXT: .LBB14_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2bf16_to_i32:
@@ -3543,13 +3549,18 @@ define float @bitcast_v2bf16_to_f32(<2 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v2bf16_to_f32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v1
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB34_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB34_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -3562,15 +3573,16 @@ define float @bitcast_v2bf16_to_f32(<2 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
-; GFX11-TRUE16-NEXT: .LBB34_2: ; %end
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11-TRUE16-NEXT: .LBB34_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2bf16_to_f32:
@@ -7051,13 +7063,18 @@ define <2 x half> @bitcast_v2bf16_to_v2f16(<2 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v2bf16_to_v2f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v1
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB62_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB62_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -7070,15 +7087,16 @@ define <2 x half> @bitcast_v2bf16_to_v2f16(<2 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
-; GFX11-TRUE16-NEXT: .LBB62_2: ; %end
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11-TRUE16-NEXT: .LBB62_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2bf16_to_v2f16:
@@ -8488,13 +8506,18 @@ define <1 x i32> @bitcast_v2bf16_to_v1i32(<2 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v2bf16_to_v1i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v1
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB72_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB72_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -8507,15 +8530,16 @@ define <1 x i32> @bitcast_v2bf16_to_v1i32(<2 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
-; GFX11-TRUE16-NEXT: .LBB72_2: ; %end
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11-TRUE16-NEXT: .LBB72_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2bf16_to_v1i32:
@@ -9062,15 +9086,14 @@ define <4 x i8> @bitcast_v2bf16_to_v4i8(<2 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
; GFX11-TRUE16-NEXT: .LBB76_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
index 68312b89142c7..fd190b23dd8ca 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
@@ -138,46 +138,51 @@ define <3 x half> @bitcast_v3bf16_to_v3f16(<3 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v3bf16_to_v3f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v2
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB0_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB0_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v2
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v6, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v3, v7 :: v_dual_and_b32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0x7fc0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h
-; GFX11-TRUE16-NEXT: .LBB0_2: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0x7fc0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT: .LBB0_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v3bf16_to_v3f16:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
index 35d135b123969..ede44e738fe00 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
@@ -5873,287 +5873,298 @@ define <16 x i32> @bitcast_v32bf16_to_v16i32(<32 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v32bf16_to_v16i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v15, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v15
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v15, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v15, v20, v22 :: v_dual_add_f32 v16, 0x40c00000, v16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v15.h
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, v15 :: v_dual_mov_b32 v30, v14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, v13 :: v_dual_mov_b32 v28, v12
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, v11 :: v_dual_mov_b32 v26, v10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, v9 :: v_dual_mov_b32 v24, v8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, v7 :: v_dual_mov_b32 v22, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, v5 :: v_dual_mov_b32 v20, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v3 :: v_dual_mov_b32 v18, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v1 :: v_dual_mov_b32 v16, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: .LBB22_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v15
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v17, 0x40c00000, v17 :: v_dual_add_f32 v16, 0x40c00000, v16
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v17, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v16, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v16
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v16
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v16, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v16, v19, v21 :: v_dual_and_b32 v17, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v17, 0x40c00000, v17 :: v_dual_lshlrev_b32 v14, 16, v14
-; GFX11-TRUE16-NEXT: v_bfi_b32 v15, 0xffff, v15, v16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v17, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v17
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v25, v14, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, 0x400000, v14
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v16, v19, v22 :: v_dual_and_b32 v15, 0xffff0000, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v15, 0x40c00000, v15 :: v_dual_lshlrev_b32 v14, 16, v14
+; GFX11-TRUE16-NEXT: v_add3_u32 v19, v23, v17, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v15, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v15
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v15, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_cndmask_b32 v31, v20, v21
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v24, v14, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v14
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_add3_u32 v20, v23, v17, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v22, v25, v14, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v16.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v17
+; GFX11-TRUE16-NEXT: v_add3_u32 v22, v24, v14, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v22, v26, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v22, v25, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v17, v20, v24 :: v_dual_and_b32 v18, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v18, 0x40c00000, v18 :: v_dual_lshlrev_b32 v13, 16, v13
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v14.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v18, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v18
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v15, 0x40c00000, v18 :: v_dual_cndmask_b32 v30, v19, v16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v15, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v14.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v13, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v13
+; GFX11-TRUE16-NEXT: v_add3_u32 v17, v18, v15, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v13
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_bfi_b32 v14, 0xffff, v14, v17
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v19, v18, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v15
; GFX11-TRUE16-NEXT: v_add3_u32 v16, v16, v13, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v13, v16, v20 :: v_dual_and_b32 v20, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v13.h
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v17, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v19, 0x40c00000, v19 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v16, v18, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_cndmask_b32 v29, v17, v19
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v13.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v12, 0x40c00000, v12 :: v_dual_lshlrev_b32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v14, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v14
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v20
-; GFX11-TRUE16-NEXT: v_bfi_b32 v13, 0xffff, v13, v16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v19, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v19
-; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v12, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v12
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v19, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v16, v16, v14, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v18, v12, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v18, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v11, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v20, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v21, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v17, v22, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v12, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v15, v17, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v13, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v28, v16, v18, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v11
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v18, v20, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v12.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v19, 0x40c00000, v22
-; GFX11-TRUE16-NEXT: v_bfi_b32 v12, 0xffff, v12, v17
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v12.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v13, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v11, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v11, v14, v16 :: v_dual_and_b32 v12, 0xffff0000, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v12, 0x40c00000, v12 :: v_dual_cndmask_b32 v27, v15, v17
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v11.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v18, v23, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v19, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v19
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v11, 0xffff, v11, v18
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v20, v19, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_lshlrev_b32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v10, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v11.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v10, 0x40c00000, v10 :: v_dual_lshlrev_b32 v9, 16, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v10
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v16, v16, v10, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v16, v20, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v17, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v9, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v12, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v10, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v13, v15, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v26, v14, v16, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v9
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v18, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v20, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v20
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v9.h
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v17, v17, v22 :: v_dual_and_b32 v22, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v10.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v9, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v12, v14 :: v_dual_and_b32 v10, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v19, v20, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v10.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v9, 0xffff, v9, v17
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v7 :: v_dual_add_f32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v10, 0x40c00000, v10 :: v_dual_cndmask_b32 v25, v13, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v9.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_lshlrev_b32 v7, 16, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v8
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v21, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v19, 0x40c00000, v22
-; GFX11-TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v10, v16
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v18, v18, v23 :: v_dual_and_b32 v23, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v19, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v8, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v11, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v24, v12, v14, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v16, v16, v7, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v19
-; GFX11-TRUE16-NEXT: v_bfi_b32 v8, 0xffff, v8, v18
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v8.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v10, v12 :: v_dual_and_b32 v8, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v20, v19, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v7, v16, v20
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v17, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v20, 0x40c00000, v20 :: v_dual_add_f32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_cndmask_b32 v23, v11, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v18, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v7.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v20, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v7, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v19, v20, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v21, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v17, v22, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v20
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v8, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v22, v10, v12, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v6, v17
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v19, 0x40c00000, v23
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v18, v22, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v19, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v6.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v9, v7, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v21, v6, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v19
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v5, v18
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v16, v16, v19, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v17, v20, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v18, 0x40c00000, v18 :: v_dual_add_f32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v19, 0x40c00000, v22
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v18
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v18, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v19, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_cndmask_b32 v4, v7, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_add_f32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v6, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v20, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v23, v19, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v19
-; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v17, v17, v22 :: v_dual_and_b32 v22, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v21, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v16
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v21, 0x40c00000, v22 :: v_dual_cndmask_b32 v18, v18, v20
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v4.h
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v19, v4, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v10, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v3.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_cndmask_b32 v2, v4, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.h
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v21
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v19, v20, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v22
-; GFX11-TRUE16-NEXT: v_bfe_u32 v22, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v21, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v20, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v22, v22, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, 0x400000, v20
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v21, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v23, v23, v20, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v22, v25, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_bfe_u32 v26, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v23, v27, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT: v_add3_u32 v22, v26, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v20.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v19, v19, v24, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v19
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v22, v25, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v20, v0
-; GFX11-TRUE16-NEXT: .LBB22_2: ; %end
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v9, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v1.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v2, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v5.h
+; GFX11-TRUE16-NEXT: .LBB22_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v16 :: v_dual_mov_b32 v1, v17
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v18 :: v_dual_mov_b32 v3, v19
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v20 :: v_dual_mov_b32 v5, v21
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v22 :: v_dual_mov_b32 v7, v23
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v24 :: v_dual_mov_b32 v9, v25
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, v26 :: v_dual_mov_b32 v11, v27
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v28 :: v_dual_mov_b32 v13, v29
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, v30 :: v_dual_mov_b32 v15, v31
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v32bf16_to_v16i32:
@@ -20624,287 +20635,298 @@ define <16 x float> @bitcast_v32bf16_to_v16f32(<32 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v32bf16_to_v16f32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v15, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v15
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v15, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v15, v20, v22 :: v_dual_add_f32 v16, 0x40c00000, v16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v15.h
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, v15 :: v_dual_mov_b32 v30, v14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, v13 :: v_dual_mov_b32 v28, v12
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, v11 :: v_dual_mov_b32 v26, v10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, v9 :: v_dual_mov_b32 v24, v8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, v7 :: v_dual_mov_b32 v22, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, v5 :: v_dual_mov_b32 v20, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v3 :: v_dual_mov_b32 v18, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v1 :: v_dual_mov_b32 v16, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: .LBB46_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v15
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v17, 0x40c00000, v17 :: v_dual_add_f32 v16, 0x40c00000, v16
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v17, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v16, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v16
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v16
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v16, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v16, v19, v21 :: v_dual_and_b32 v17, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v17, 0x40c00000, v17 :: v_dual_lshlrev_b32 v14, 16, v14
-; GFX11-TRUE16-NEXT: v_bfi_b32 v15, 0xffff, v15, v16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v17, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v17
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v25, v14, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, 0x400000, v14
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v16, v19, v22 :: v_dual_and_b32 v15, 0xffff0000, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v15, 0x40c00000, v15 :: v_dual_lshlrev_b32 v14, 16, v14
+; GFX11-TRUE16-NEXT: v_add3_u32 v19, v23, v17, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v15, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v15
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v15, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_cndmask_b32 v31, v20, v21
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v24, v14, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v14
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_add3_u32 v20, v23, v17, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v22, v25, v14, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v16.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v17
+; GFX11-TRUE16-NEXT: v_add3_u32 v22, v24, v14, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v22, v26, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v22, v25, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v17, v20, v24 :: v_dual_and_b32 v18, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v18, 0x40c00000, v18 :: v_dual_lshlrev_b32 v13, 16, v13
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v14.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v18, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v18
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v15, 0x40c00000, v18 :: v_dual_cndmask_b32 v30, v19, v16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v15, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v14.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v13, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v13
+; GFX11-TRUE16-NEXT: v_add3_u32 v17, v18, v15, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v13
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_bfi_b32 v14, 0xffff, v14, v17
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v19, v18, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v15
; GFX11-TRUE16-NEXT: v_add3_u32 v16, v16, v13, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v13, v16, v20 :: v_dual_and_b32 v20, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v13.h
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v17, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v19, 0x40c00000, v19 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v16, v18, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_cndmask_b32 v29, v17, v19
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v13.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v12, 0x40c00000, v12 :: v_dual_lshlrev_b32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v14, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v14
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v20
-; GFX11-TRUE16-NEXT: v_bfi_b32 v13, 0xffff, v13, v16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v19, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v19
-; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v12, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v12
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v19, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v16, v16, v14, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v18, v12, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v18, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v11, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v20, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v21, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v17, v22, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v12, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v15, v17, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v13, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v28, v16, v18, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v11
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v18, v20, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v12.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v19, 0x40c00000, v22
-; GFX11-TRUE16-NEXT: v_bfi_b32 v12, 0xffff, v12, v17
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v12.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v13, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v11, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v11, v14, v16 :: v_dual_and_b32 v12, 0xffff0000, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v12, 0x40c00000, v12 :: v_dual_cndmask_b32 v27, v15, v17
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v11.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v18, v23, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v19, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v19
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v11, 0xffff, v11, v18
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v20, v19, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_lshlrev_b32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v10, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v11.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v10, 0x40c00000, v10 :: v_dual_lshlrev_b32 v9, 16, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v10
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v16, v16, v10, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v16, v20, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v17, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v9, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v12, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v10, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v13, v15, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v26, v14, v16, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v9
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v18, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v20, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v20
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v9.h
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v17, v17, v22 :: v_dual_and_b32 v22, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v10.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v9, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v12, v14 :: v_dual_and_b32 v10, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v19, v20, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v10.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v9, 0xffff, v9, v17
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v7 :: v_dual_add_f32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v10, 0x40c00000, v10 :: v_dual_cndmask_b32 v25, v13, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v9.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_lshlrev_b32 v7, 16, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v8
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v21, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v19, 0x40c00000, v22
-; GFX11-TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v10, v16
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v18, v18, v23 :: v_dual_and_b32 v23, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v19, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v8, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v11, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v24, v12, v14, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v16, v16, v7, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v19
-; GFX11-TRUE16-NEXT: v_bfi_b32 v8, 0xffff, v8, v18
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v8.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v10, v12 :: v_dual_and_b32 v8, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v20, v19, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v7, v16, v20
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v17, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v20, 0x40c00000, v20 :: v_dual_add_f32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_cndmask_b32 v23, v11, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v18, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v7.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v20, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v7, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v19, v20, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v21, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v17, v22, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v20
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v8, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v22, v10, v12, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v6, v17
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v19, 0x40c00000, v23
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v18, v22, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v19, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v6.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v9, v7, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v21, v6, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v19
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v5, v18
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v16, v16, v19, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v17, v20, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v18, 0x40c00000, v18 :: v_dual_add_f32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v19, 0x40c00000, v22
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v18
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v18, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v19, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_cndmask_b32 v4, v7, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_add_f32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v6, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v20, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v23, v19, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v19
-; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v17, v17, v22 :: v_dual_and_b32 v22, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v21, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v16
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v21, 0x40c00000, v22 :: v_dual_cndmask_b32 v18, v18, v20
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v4.h
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v19, v4, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v10, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v3.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_cndmask_b32 v2, v4, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.h
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v21
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v19, v20, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v22
-; GFX11-TRUE16-NEXT: v_bfe_u32 v22, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v21, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v20, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v22, v22, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, 0x400000, v20
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v21, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v23, v23, v20, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v22, v25, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_bfe_u32 v26, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v23, v27, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT: v_add3_u32 v22, v26, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v20.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v19, v19, v24, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v19
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v22, v25, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v20, v0
-; GFX11-TRUE16-NEXT: .LBB46_2: ; %end
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v9, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v1.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v2, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v5.h
+; GFX11-TRUE16-NEXT: .LBB46_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v16 :: v_dual_mov_b32 v1, v17
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v18 :: v_dual_mov_b32 v3, v19
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v20 :: v_dual_mov_b32 v5, v21
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v22 :: v_dual_mov_b32 v7, v23
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v24 :: v_dual_mov_b32 v9, v25
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, v26 :: v_dual_mov_b32 v11, v27
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v28 :: v_dual_mov_b32 v13, v29
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, v30 :: v_dual_mov_b32 v15, v31
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v32bf16_to_v16f32:
@@ -34883,287 +34905,298 @@ define <8 x i64> @bitcast_v32bf16_to_v8i64(<32 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v32bf16_to_v8i64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB66_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v15, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v15
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v15, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v15, v20, v22 :: v_dual_add_f32 v16, 0x40c00000, v16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v15.h
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, v15 :: v_dual_mov_b32 v30, v14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, v13 :: v_dual_mov_b32 v28, v12
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, v11 :: v_dual_mov_b32 v26, v10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, v9 :: v_dual_mov_b32 v24, v8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, v7 :: v_dual_mov_b32 v22, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, v5 :: v_dual_mov_b32 v20, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v3 :: v_dual_mov_b32 v18, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v1 :: v_dual_mov_b32 v16, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: .LBB66_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB66_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v15
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v17, 0x40c00000, v17 :: v_dual_add_f32 v16, 0x40c00000, v16
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v17, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v16, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v16
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v16
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v16, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v16, v19, v21 :: v_dual_and_b32 v17, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v17, 0x40c00000, v17 :: v_dual_lshlrev_b32 v14, 16, v14
-; GFX11-TRUE16-NEXT: v_bfi_b32 v15, 0xffff, v15, v16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v17, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v17
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v25, v14, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, 0x400000, v14
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v16, v19, v22 :: v_dual_and_b32 v15, 0xffff0000, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v15, 0x40c00000, v15 :: v_dual_lshlrev_b32 v14, 16, v14
+; GFX11-TRUE16-NEXT: v_add3_u32 v19, v23, v17, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v15, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v15
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v15, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_cndmask_b32 v31, v20, v21
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v24, v14, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v14
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_add3_u32 v20, v23, v17, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v22, v25, v14, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v16.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v17
+; GFX11-TRUE16-NEXT: v_add3_u32 v22, v24, v14, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v22, v26, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v22, v25, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v17, v20, v24 :: v_dual_and_b32 v18, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v18, 0x40c00000, v18 :: v_dual_lshlrev_b32 v13, 16, v13
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v14.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v18, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v18
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v15, 0x40c00000, v18 :: v_dual_cndmask_b32 v30, v19, v16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v15, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v14.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v13, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v13
+; GFX11-TRUE16-NEXT: v_add3_u32 v17, v18, v15, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v13
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_bfi_b32 v14, 0xffff, v14, v17
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v19, v18, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v15
; GFX11-TRUE16-NEXT: v_add3_u32 v16, v16, v13, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v13, v16, v20 :: v_dual_and_b32 v20, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v13.h
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v17, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v19, 0x40c00000, v19 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v16, v18, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_cndmask_b32 v29, v17, v19
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v13.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v12, 0x40c00000, v12 :: v_dual_lshlrev_b32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v14, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v14
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v20
-; GFX11-TRUE16-NEXT: v_bfi_b32 v13, 0xffff, v13, v16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v19, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v19
-; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v12, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v12
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v19, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v16, v16, v14, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v18, v12, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v18, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v11, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v20, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v21, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v17, v22, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v12, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v15, v17, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v13, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v28, v16, v18, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v11
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v18, v20, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v12.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v19, 0x40c00000, v22
-; GFX11-TRUE16-NEXT: v_bfi_b32 v12, 0xffff, v12, v17
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v12.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v13, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v11, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v11, v14, v16 :: v_dual_and_b32 v12, 0xffff0000, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v12, 0x40c00000, v12 :: v_dual_cndmask_b32 v27, v15, v17
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v11.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v18, v23, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v19, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v19
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v11, 0xffff, v11, v18
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v20, v19, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_lshlrev_b32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v10, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v11.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v10, 0x40c00000, v10 :: v_dual_lshlrev_b32 v9, 16, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v10
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v16, v16, v10, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v16, v20, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v17, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v9, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v12, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v10, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v13, v15, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v26, v14, v16, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v9
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v18, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v20, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v20
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v9.h
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v17, v17, v22 :: v_dual_and_b32 v22, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v10.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v9, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v12, v14 :: v_dual_and_b32 v10, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v19, v20, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v10.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v9, 0xffff, v9, v17
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v7 :: v_dual_add_f32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v10, 0x40c00000, v10 :: v_dual_cndmask_b32 v25, v13, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v9.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_lshlrev_b32 v7, 16, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v8
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v21, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v19, 0x40c00000, v22
-; GFX11-TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v10, v16
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v18, v18, v23 :: v_dual_and_b32 v23, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v19, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v8, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v11, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v24, v12, v14, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v16, v16, v7, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v19
-; GFX11-TRUE16-NEXT: v_bfi_b32 v8, 0xffff, v8, v18
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v8.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v10, v12 :: v_dual_and_b32 v8, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v20, v19, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v7, v16, v20
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v17, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v20, 0x40c00000, v20 :: v_dual_add_f32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_cndmask_b32 v23, v11, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v18, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v7.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v20, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v7, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v19, v20, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v21, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v17, v22, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v20
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v8, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v22, v10, v12, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v6, v17
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v19, 0x40c00000, v23
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v18, v22, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v19, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v6.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v9, v7, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v21, v6, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v19
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v5, v18
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v16, v16, v19, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v17, v20, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v18, 0x40c00000, v18 :: v_dual_add_f32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v19, 0x40c00000, v22
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v18
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v18, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v19, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_cndmask_b32 v4, v7, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_add_f32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v6, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v20, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v23, v19, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v19
-; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v17, v17, v22 :: v_dual_and_b32 v22, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v21, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v16
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v21, 0x40c00000, v22 :: v_dual_cndmask_b32 v18, v18, v20
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v4.h
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v19, v4, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v10, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v3.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_cndmask_b32 v2, v4, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.h
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v21
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v19, v20, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v22
-; GFX11-TRUE16-NEXT: v_bfe_u32 v22, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v21, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v20, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v22, v22, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, 0x400000, v20
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v21, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v23, v23, v20, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v22, v25, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_bfe_u32 v26, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v23, v27, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT: v_add3_u32 v22, v26, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v20.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v19, v19, v24, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v19
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v22, v25, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v20, v0
-; GFX11-TRUE16-NEXT: .LBB66_2: ; %end
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v9, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v1.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v2, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v5.h
+; GFX11-TRUE16-NEXT: .LBB66_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v16 :: v_dual_mov_b32 v1, v17
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v18 :: v_dual_mov_b32 v3, v19
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v20 :: v_dual_mov_b32 v5, v21
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v22 :: v_dual_mov_b32 v7, v23
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v24 :: v_dual_mov_b32 v9, v25
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, v26 :: v_dual_mov_b32 v11, v27
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v28 :: v_dual_mov_b32 v13, v29
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, v30 :: v_dual_mov_b32 v15, v31
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v32bf16_to_v8i64:
@@ -48217,287 +48250,298 @@ define <8 x double> @bitcast_v32bf16_to_v8f64(<32 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v32bf16_to_v8f64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v15, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v15
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v15, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v15, v20, v22 :: v_dual_add_f32 v16, 0x40c00000, v16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v15.h
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, v15 :: v_dual_mov_b32 v30, v14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, v13 :: v_dual_mov_b32 v28, v12
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, v11 :: v_dual_mov_b32 v26, v10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, v9 :: v_dual_mov_b32 v24, v8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, v7 :: v_dual_mov_b32 v22, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, v5 :: v_dual_mov_b32 v20, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v3 :: v_dual_mov_b32 v18, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v1 :: v_dual_mov_b32 v16, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: .LBB82_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v15
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v17, 0x40c00000, v17 :: v_dual_add_f32 v16, 0x40c00000, v16
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v17, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v16, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v16
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v16
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v16, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v16, v19, v21 :: v_dual_and_b32 v17, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v17, 0x40c00000, v17 :: v_dual_lshlrev_b32 v14, 16, v14
-; GFX11-TRUE16-NEXT: v_bfi_b32 v15, 0xffff, v15, v16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v17, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v17
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v25, v14, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, 0x400000, v14
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v16, v19, v22 :: v_dual_and_b32 v15, 0xffff0000, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v15, 0x40c00000, v15 :: v_dual_lshlrev_b32 v14, 16, v14
+; GFX11-TRUE16-NEXT: v_add3_u32 v19, v23, v17, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v15, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v15
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v15, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_cndmask_b32 v31, v20, v21
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v24, v14, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v14
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_add3_u32 v20, v23, v17, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v22, v25, v14, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v16.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v17
+; GFX11-TRUE16-NEXT: v_add3_u32 v22, v24, v14, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v22, v26, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v22, v25, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v17, v20, v24 :: v_dual_and_b32 v18, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v18, 0x40c00000, v18 :: v_dual_lshlrev_b32 v13, 16, v13
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v14.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v18, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v18
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v15, 0x40c00000, v18 :: v_dual_cndmask_b32 v30, v19, v16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v15, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v14.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v13, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v13
+; GFX11-TRUE16-NEXT: v_add3_u32 v17, v18, v15, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v13
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_bfi_b32 v14, 0xffff, v14, v17
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v19, v18, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v15
; GFX11-TRUE16-NEXT: v_add3_u32 v16, v16, v13, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v13, v16, v20 :: v_dual_and_b32 v20, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v13.h
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v17, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v19, 0x40c00000, v19 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v16, v18, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_cndmask_b32 v29, v17, v19
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v13.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v12, 0x40c00000, v12 :: v_dual_lshlrev_b32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v14, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v14
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v20
-; GFX11-TRUE16-NEXT: v_bfi_b32 v13, 0xffff, v13, v16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v19, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v19
-; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v12, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v12
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v19, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v16, v16, v14, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v18, v12, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v18, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v11, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v20, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v21, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v17, v22, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v12, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v15, v17, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v13, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v28, v16, v18, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v11
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v18, v20, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v12.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v19, 0x40c00000, v22
-; GFX11-TRUE16-NEXT: v_bfi_b32 v12, 0xffff, v12, v17
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v12.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v13, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v11, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v11, v14, v16 :: v_dual_and_b32 v12, 0xffff0000, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v12, 0x40c00000, v12 :: v_dual_cndmask_b32 v27, v15, v17
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v11.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v18, v23, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v19, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v19
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v11, 0xffff, v11, v18
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v20, v19, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_lshlrev_b32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v10, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v11.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v10, 0x40c00000, v10 :: v_dual_lshlrev_b32 v9, 16, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v10
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v16, v16, v10, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v16, v20, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v17, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v9, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v12, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v10, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v13, v15, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v26, v14, v16, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v9
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v18, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v20, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v20
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v9.h
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v17, v17, v22 :: v_dual_and_b32 v22, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v10.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v9, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v12, v14 :: v_dual_and_b32 v10, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v19, v20, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v10.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v9, 0xffff, v9, v17
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v7 :: v_dual_add_f32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v10, 0x40c00000, v10 :: v_dual_cndmask_b32 v25, v13, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v9.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_lshlrev_b32 v7, 16, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v8
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v21, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v19, 0x40c00000, v22
-; GFX11-TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v10, v16
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v18, v18, v23 :: v_dual_and_b32 v23, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v19, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v8, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v11, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v24, v12, v14, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v16, v16, v7, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v19
-; GFX11-TRUE16-NEXT: v_bfi_b32 v8, 0xffff, v8, v18
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v8.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v10, v12 :: v_dual_and_b32 v8, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v20, v19, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v7, v16, v20
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v17, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v20, 0x40c00000, v20 :: v_dual_add_f32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_cndmask_b32 v23, v11, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v18, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v7.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v20, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v7, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v19, v20, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v21, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v17, v22, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v20
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v8, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v22, v10, v12, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v6, v17
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v19, 0x40c00000, v23
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v18, v22, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v19, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v6.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v9, v7, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v21, v6, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v5.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v19
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v5, v18
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v16, v16, v19, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v17, v20, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v18, 0x40c00000, v18 :: v_dual_add_f32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v19, 0x40c00000, v22
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v18
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v18, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v19, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_cndmask_b32 v4, v7, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_add_f32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v6, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v20, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v23, v19, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v19
-; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v17, v17, v22 :: v_dual_and_b32 v22, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v21, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v16
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v21, 0x40c00000, v22 :: v_dual_cndmask_b32 v18, v18, v20
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v4.h
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v19, v4, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v10, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v3.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_cndmask_b32 v2, v4, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.h
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v21
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v19, v20, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v22
-; GFX11-TRUE16-NEXT: v_bfe_u32 v22, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v21, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v20, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v22, v22, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, 0x400000, v20
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v21, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v23, v23, v20, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v22, v25, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_bfe_u32 v26, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v23, v27, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT: v_add3_u32 v22, v26, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v20.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v19, v19, v24, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v19
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v22, v25, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v20, v0
-; GFX11-TRUE16-NEXT: .LBB82_2: ; %end
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v9, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v1.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v2, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v5.h
+; GFX11-TRUE16-NEXT: .LBB82_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v16 :: v_dual_mov_b32 v1, v17
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v18 :: v_dual_mov_b32 v3, v19
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v20 :: v_dual_mov_b32 v5, v21
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v22 :: v_dual_mov_b32 v7, v23
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v24 :: v_dual_mov_b32 v9, v25
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, v26 :: v_dual_mov_b32 v11, v27
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v28 :: v_dual_mov_b32 v13, v29
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, v30 :: v_dual_mov_b32 v15, v31
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v32bf16_to_v8f64:
@@ -73032,283 +73076,291 @@ define <32 x half> @bitcast_v32bf16_to_v32f16(<32 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v32bf16_to_v32f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v32, v15 :: v_dual_mov_b32 v31, v14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v30, v13 :: v_dual_mov_b32 v29, v12
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v28, v11 :: v_dual_mov_b32 v27, v10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, v9 :: v_dual_mov_b32 v25, v8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v24, v7 :: v_dual_mov_b32 v23, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v22, v5 :: v_dual_mov_b32 v21, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v20, v3 :: v_dual_mov_b32 v19, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, v1 :: v_dual_mov_b32 v17, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB102_3
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
+; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB102_4
+; GFX11-TRUE16-NEXT: .LBB102_2: ; %end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-NEXT: .LBB102_3: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v17 :: v_dual_mov_b32 v1, v18
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v19 :: v_dual_mov_b32 v3, v20
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v21 :: v_dual_mov_b32 v5, v22
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v23 :: v_dual_mov_b32 v7, v24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v25 :: v_dual_mov_b32 v9, v26
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, v27 :: v_dual_mov_b32 v11, v28
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v29 :: v_dual_mov_b32 v13, v30
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, v31 :: v_dual_mov_b32 v15, v32
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB102_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xffff0000, v8
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xffff0000, v10
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v16
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v17, 0x40c00000, v17 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v19, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v16, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; GFX11-TRUE16-NEXT: v_bfe_u32 v22, v17, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v19, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v16, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v19
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v17
-; GFX11-TRUE16-NEXT: v_add3_u32 v22, v22, v17, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v21, v21, v19, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v20, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v16, v21, v23 :: v_dual_add_f32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v17, v22, v24 :: v_dual_add_f32 v18, 0x40c00000, v18
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v20
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v22, v18, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: .LBB102_4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v18
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v19
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v23
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v25
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v31
; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v20, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v22, v18, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v18
-; GFX11-TRUE16-NEXT: v_bfe_u32 v22, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v16.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v22, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v24
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v26
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v17
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v1, 16, v17
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_cndmask_b32 v0, v5, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v20
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v18
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v16, v7, v9 :: v_dual_add_f32 v3, 0x40c00000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v22, v23, v20, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v20
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v17
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v22, v23, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v20
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v16.h
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v8, v10 :: v_dual_and_b32 v10, 0xffff0000, v21
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v19
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v19, v23, vcc_lo
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v21, 0x40c00000, v24 :: v_dual_add_f32 v22, 0x40c00000, v22
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v24, v21, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v21
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT: v_bfe_u32 v25, v22, 16, 1
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v20
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v24, v21, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v19, v19, v23 :: v_dual_lshlrev_b32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v24, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_cndmask_b32 v17, v5, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v8, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v17.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v7, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v21, v24, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v24, v25, v22, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v22
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v21, v23, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v23, 0x40c00000, v26
-; GFX11-TRUE16-NEXT: v_add3_u32 v21, v21, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v22, v24, v25, vcc_lo
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v26, v23, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v24, 0x40c00000, v24 :: v_dual_lshlrev_b32 v7, 16, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v21, v25, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v21, v26, v23, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v23
-; GFX11-TRUE16-NEXT: v_bfe_u32 v26, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11-TRUE16-NEXT: v_bfe_u32 v27, v24, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v23, v26, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v21, v21, v25, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v3, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v26, v27, v24, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, 0x400000, v24
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v5, v22
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v23, v25, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v25, 0x40c00000, v28
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v10 :: v_dual_lshlrev_b32 v6, 16, v21
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v22
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v8, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v23, v23, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v24, v26, v27, vcc_lo
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff0000, v9
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_bfe_u32 v28, v25, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_add_f32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v19, v4, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v10, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v19.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v9, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v10, v7, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v10, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v26, 0x40c00000, v26 :: v_dual_lshlrev_b32 v9, 16, v9
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v19
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v23, v27, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v23, v28, v25, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, 0x400000, v25
-; GFX11-TRUE16-NEXT: v_bfe_u32 v28, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT: v_bfe_u32 v29, v26, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v12 :: v_dual_lshlrev_b32 v8, 16, v23
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v9, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v24
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v7.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v25, v28, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v23, v23, v27, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v27
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v18.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v21, v6, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v12, v7, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v21.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v11, v8, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v8
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_add3_u32 v28, v29, v26, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, 0x400000, v26
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v25, v27, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v25, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v27, 0x40c00000, v30
-; GFX11-TRUE16-NEXT: v_add3_u32 v25, v25, v9, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v26, v28, v29, vcc_lo
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_bfe_u32 v30, v27, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v12, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v13
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v22, v7, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v14 :: v_dual_lshlrev_b32 v10, 16, v25
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v11, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v26
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v12, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v9, 16, 1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v28, 0x40c00000, v28 :: v_dual_lshlrev_b32 v11, 16, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v25, v29, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v25, v30, v27, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, 0x400000, v27
-; GFX11-TRUE16-NEXT: v_bfe_u32 v30, v10, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT: v_bfe_u32 v31, v28, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v9.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v27, v30, v10, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v25, v25, v29, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v29, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v10, 0x40c00000, v10 :: v_dual_add_f32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v20.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v23, v8, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v14, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v23.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v13, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v14, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v14, 0x40c00000, v15
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_add3_u32 v30, v31, v28, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, 0x400000, v28
-; GFX11-TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v6, v21
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v27, v29, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v27, v11, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v29, 0x40c00000, v32
-; GFX11-TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v7, v24
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v11
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v27, v27, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v28, v30, v31, vcc_lo
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v30, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, 0x400000, v11
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v29, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v25, v9, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v27
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v28
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v14, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v24
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v13, v15, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v14
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v14, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v28
+; GFX11-TRUE16-NEXT: v_bfe_u32 v24, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v11
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v30, 0x40c00000, v30 :: v_dual_lshlrev_b32 v13, 16, v13
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v27, v31, vcc_lo
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v26, v10, v15 :: v_dual_add_f32 v13, 0x40c00000, v13
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v24, v11, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v27, v32, v29, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, 0x400000, v29
-; GFX11-TRUE16-NEXT: v_bfe_u32 v32, v12, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v30, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v11.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v29, v32, v12, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v27, v27, v31, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xffff0000, v29
+; GFX11-TRUE16-NEXT: v_bfe_u32 v24, v13, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v26.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v15, v12, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v10, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_add3_u32 v15, v24, v13, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v24, 0x40c00000, v27
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_add3_u32 v32, v33, v30, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v30
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v34, 0x400000, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v29, v31, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v31, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11-TRUE16-NEXT: v_bfe_u32 v29, v13, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v12.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v30, v32, v33, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v15
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_add_f32 v31, 0x40c00000, v31
-; GFX11-TRUE16-NEXT: v_add3_u32 v29, v29, v13, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v32, 0x40c00000, v32 :: v_dual_add_f32 v15, 0x40c00000, v15
-; GFX11-TRUE16-NEXT: v_bfe_u32 v35, v14, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v48, 0x400000, v14
-; GFX11-TRUE16-NEXT: v_bfe_u32 v33, v31, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, 0x400000, v13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v25.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v22.h
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v24, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v11, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v13, 0x40c00000, v28 :: v_dual_lshlrev_b32 v14, 16, v29
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v24, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v33.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v15, v27, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, 0x400000, v24
+; GFX11-TRUE16-NEXT: v_bfe_u32 v28, v13, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v30
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v30
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v24, v12, v27, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v28, v13, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, 0x400000, v13
+; GFX11-TRUE16-NEXT: v_bfe_u32 v28, v14, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-TRUE16-NEXT: v_bfe_u32 v29, v15, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v24.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v36, v32, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v39, 0x400000, v32
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32
-; GFX11-TRUE16-NEXT: v_bfe_u32 v38, v15, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v35, v35, v14, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v36, v36, v32, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v49, 0x400000, v15
-; GFX11-TRUE16-NEXT: v_add3_u32 v33, v33, v31, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v38, v38, v15, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v31
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v36, v39, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v28, v14, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v12, v27, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v27, 0x400000, v14
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v10.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v12, 0xffff, v12, v27
-; GFX11-TRUE16-NEXT: v_bfi_b32 v11, 0xffff, v11, v28
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v32.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v35, v48, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v28, v29, v15, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v29, 0x40c00000, v30
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v30, 0x400000, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v27, v13, v27, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v10, v25
-; GFX11-TRUE16-NEXT: v_bfi_b32 v9, 0xffff, v9, v26
-; GFX11-TRUE16-NEXT: v_bfi_b32 v8, 0xffff, v8, v23
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v14.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v15, v38, v49, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v15, 0xffff, v32, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v29, v34, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v31, v31
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v13.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v29, v33, v37, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v13, 0xffff, v13, v30
-; GFX11-TRUE16-NEXT: v_bfi_b32 v14, 0xffff, v14, v29
-; GFX11-TRUE16-NEXT: .LBB102_2: ; %end
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v29, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v15, 0x40c00000, v34
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v27.h
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v13, v28, v30 :: v_dual_lshlrev_b32 v28, 16, v31
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v29, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v32
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v31, 0x400000, v29
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v28, 0x40c00000, v28
+; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v15, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v30, 0x40c00000, v30
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v29, v14, v31 :: v_dual_and_b32 v32, 0xffff0000, v32
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v28, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v31, v34, v15, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v34, v30, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v28
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
+; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v28, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v32, 0x40c00000, v32
+; GFX11-TRUE16-NEXT: v_add3_u32 v34, v34, v30, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v30
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v35, 0x400000, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v28, v14, v36, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
+; GFX11-TRUE16-NEXT: v_bfe_u32 v37, v32, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v29.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v30, v34, v38, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v36, v37, v32, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v37, 0x400000, v32
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v31, v35, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v28.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v15, v36, v37, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v30.h
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -85371,559 +85423,560 @@ define <64 x i8> @bitcast_v32bf16_to_v64i8(<32 x bfloat> %a, i32 %b) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v17
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr131_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr130_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr129_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr128_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr119_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr118_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr117_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr116_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr115_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr113_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr102_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr101_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr100_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr99_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr98_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr97_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr112_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr84_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr82_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr96_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr81_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr80_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr114_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr85_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr87_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr83_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr86_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr103_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[17:18], 24, v[15:16]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[13:14]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[20:21], 24, v[9:10]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[21:22], 24, v[7:8]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[22:23], 24, v[5:6]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[23:24], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 24, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v50, 8, v15
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 24, v14
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 8, v14
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v64, 8, v13
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v65, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v66, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v68, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 24, v10
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 8, v10
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 8, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 24, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 8, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 8, v7
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v97, 24, v6
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 8, v6
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 8, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 24, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 8, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[24:25], 24, v[1:2]
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.h, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.h, v4.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.h, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.h, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.h, v6.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.h, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.h, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v8.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.h, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.h, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.h, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.h, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.h, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.h, v12.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.h, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.h, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.h, v14.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.h, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.h, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.h, v16.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[48:49], 24, v[7:8]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[49:50], 24, v[5:6]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[29:30], 24, v[15:16]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[11:12]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[50:51], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v69, 24, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 8, v15
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 24, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 8, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 8, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v85, 24, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v86, 8, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 24, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 24, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 8, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 24, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 8, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v117, 8, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v118, 24, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v128, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v129, 24, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v131, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[30:31], 24, v[13:14]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[9:10]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[51:52], 24, v[1:2]
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v1.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v2.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v4.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.h, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v5.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v6.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.h, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v7.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.h, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v8.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.h, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v9.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.h, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v10.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.h, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v11.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.h, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v12.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v112.h, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.h, v13.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.h, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.h, v14.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.h, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.h, v15.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v103.h, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.h, v16.h
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15
; GFX11-TRUE16-NEXT: .LBB108_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v18, 0x40c00000, v18 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v11, 16, v11
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v17, 0x40c00000, v17
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v17, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v17
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v17, 0x40c00000, v17 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v20, 0x40c00000, v18
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v17, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v17
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT: v_add3_u32 v21, v21, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v17, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v26, v20, v22, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v18, v18, v17, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v39, v18, v23 :: v_dual_and_b32 v2, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v18
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v18, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v26.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v27, v21, v23, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v19, 0x40c00000, v19
+; GFX11-TRUE16-NEXT: v_bfe_u32 v24, v20, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v25, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v21, v21, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, 0x400000, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v27
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v28, v20, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v19
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v28.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v17, v22, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v18, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v19, 16, 1
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v20, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v29, v18, v22, vcc_lo
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v17, v24, v20, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v23, v25, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v21, v22, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v19
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v20
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v39.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v53, v23, v26, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v19, 0x7fff
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v21, 0x40c00000, v21 :: v_dual_lshlrev_b32 v6, 16, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v20, 0x40c00000, v20
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v30, v17, v23, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v21, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v21
-; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v17, v19, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v22, 0x40c00000, v20
+; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v53.h
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v22, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v54, v19, v21, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v22
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v22, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v129, 24, v18
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v1, v20, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v3, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v21, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v29.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v18, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_add_f32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 24, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 8, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v31, v18, v19, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v18, 0x40c00000, v22
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v31.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v23, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v18, 16, 1
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v17, v30
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v19, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v21, v18, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v54.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 8, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v131, 8, v17
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v55, v1, v19 :: v_dual_and_b32 v2, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v19, v4, v21, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v55.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v118, 24, v20
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 8, v20
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v128, 8, v19
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v6, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v18
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v20, 16, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 8, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v17, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v20, 0x7fff
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 8, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v19, v22, vcc_lo
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v64, v3, v21 :: v_dual_and_b32 v3, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v22, v1, v22, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v20
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v32.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v34, v17, v21 :: v_dual_add_f32 v19, 0x40c00000, v19
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v34.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v19, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v6, v22, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v17, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v19, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v19
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v35, v6, v17, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v18, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v36, v5, v22, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v21, v20
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v10
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v64.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v65, v2, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v5, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v21, v1, v21, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v2, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v35.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v97, 24, v6
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 8, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v10
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v65.h
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v66, v4, v5 :: v_dual_lshlrev_b32 v5, 16, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v20, 0x40c00000, v20 :: v_dual_add_f32 v7, 0x40c00000, v7
-; GFX11-TRUE16-NEXT: v_bfi_b32 v8, 0xffff, v8, v36
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 8, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_add_f32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v24, v1, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v66.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 24, v22
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 8, v22
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v117, 8, v21
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v23, v1, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v82, 24, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 8, v8
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v20, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v38, v19, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v20, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v20
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v17, 0x40c00000, v23 :: v_dual_add_f32 v10, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v17, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v17
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v10, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v18, v17, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v10, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v18, v22, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v38.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v37, v7, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v12
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v39, v19, v22, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v21, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v37.h
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; GFX11-TRUE16-NEXT: v_bfe_u32 v22, v21, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v48, 0x400000, v21
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v19, v39
-; GFX11-TRUE16-NEXT: v_bfe_u32 v24, v12, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v22, v22, v21, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v9
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v50, 0x400000, v12
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v21, 0x40c00000, v49
-; GFX11-TRUE16-NEXT: v_add3_u32 v24, v24, v12, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v52, v22, v48 :: v_dual_add_f32 v9, 0x40c00000, v23
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v7 :: v_dual_lshlrev_b32 v14, 16, v14
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 24, v10
-; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v9, 0x7fff
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 8, v10
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v19, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v52.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v55, v19, v25, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v21, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v25, v14, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v53, v24, v50, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v12, 0xffff, v7, v53
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v20, v23, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v11
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v19, v21, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v21
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v55.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v65, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v66, 8, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v11, v19, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v19, 0x40c00000, v22
-; GFX11-TRUE16-NEXT: v_add3_u32 v21, v23, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_bfe_u32 v24, v19, 16, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT: v_bfi_b32 v9, 0xffff, v20, v9
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v23 :: v_dual_cndmask_b32 v70, v21, v22
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v68, v3, v8 :: v_dual_and_b32 v3, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v12
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v67, v1, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v9
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v26, v2, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v68.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v67.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v12
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[48:49], 24, v[23:24]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[49:50], 24, v[21:22]
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v25, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v7, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[50:51], 24, v[19:20]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[51:52], 24, v[17:18]
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v6, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v83, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v8 :: v_dual_lshlrev_b32 v5, 16, v14
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v21, v24, v19, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v19
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_add3_u32 v23, v25, v14, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v14
-; GFX11-TRUE16-NEXT: v_bfe_u32 v25, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v13
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v67, v21, v22, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_add3_u32 v14, v25, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v16
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v69, v23, v24, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v13, 16, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 8, v9
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v69.h
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v14, v19 :: v_dual_add_f32 v14, 0x40c00000, v21
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v15
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v23, v13, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v13
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v16, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v21, 0x40c00000, v21
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v15
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v16
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v85, v19, v23, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v16, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v21, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT: v_bfe_u32 v24, v14, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v15, 0x40c00000, v15
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v49, 0x400000, v21
-; GFX11-TRUE16-NEXT: v_add3_u32 v23, v23, v21, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v83, v13, v25, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11-TRUE16-NEXT: v_add3_u32 v19, v24, v14, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v14
-; GFX11-TRUE16-NEXT: v_bfe_u32 v48, v15, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v87, v23, v49, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v85.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v13, v48, v15, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v70.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v14, 0xffff, v22, v67
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v86, v19, v24, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v83.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v87.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v11, 0xffff, v23, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 24, v14
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v13, v16, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v16, 0xffff, v19, v86
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 8, v14
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v68, 8, v11
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v83.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v101, 24, v24
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v82, v1, v2 :: v_dual_lshlrev_b32 v1, 16, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 8, v25
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v7, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v98, 24, v26
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 8, v26
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v28, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v102, 8, v24
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v113, 8, v23
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v27, v2, v3 :: v_dual_add_f32 v2, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v82.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v97, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v97.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v13
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v8, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v96, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[27:28]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[25:26]
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v96.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v85, 24, v28
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v32, v3, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v16
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 8, v33
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v86, 8, v28
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v112, v4, v6 :: v_dual_add_f32 v1, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v15, 0xffff, v15, v13
-; GFX11-TRUE16-NEXT: v_bfi_b32 v13, 0xffff, v21, v7
-; GFX11-TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v18, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 24, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[17:18], 24, v[15:16]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[13:14]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[20:21], 24, v[9:10]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[21:22], 24, v[7:8]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[22:23], 24, v[5:6]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[23:24], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[24:25], 24, v[1:2]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v50, 8, v15
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v64, 8, v13
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 8, v7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v112.h
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v103, v2, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v15
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v10, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 8, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 8, v27
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v114, v7, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v38, v4, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v103.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v37, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v114.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v69, 24, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v70, 8, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[29:30], 24, v[37:38]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[30:31], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 8, v37
; GFX11-TRUE16-NEXT: .LBB108_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v28.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v113.l
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, 0
-; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v27.h
-; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v1.l, v2.l
-; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v26.h
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v1.h, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v24.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v112.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v103.l
-; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v23.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v24, v1
-; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v3.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v24.h
-; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v31.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v102.l
-; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v3.h, v4.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v101.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v24, v2
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v30.h
-; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v3.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v24.h
-; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v29.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v100.l
-; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v22.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v24, v3
-; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v5.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v24.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v34.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v99.l
-; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v5.h, v6.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v98.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v24, v4
-; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v33.h
-; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v5.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v24.h
-; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v32.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v97.l
-; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v21.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v24, v5
-; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v6.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v24.h
-; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v38.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v96.l
-; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v7.h, v8.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v84.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v24, v6
-; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v36.h
-; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v7.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v24.h
-; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v35.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v82.l
-; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v9.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v20.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v24, v7
-; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v8.l, v8.h
-; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v9.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v24.h
-; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v55.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v81.l
-; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v9.h, v10.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v80.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v24, v8
-; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v39.h
-; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v9.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v24.h
-; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v37.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v71.l
-; GFX11-TRUE16-NEXT: v_and_b16 v11.h, 0xff, v11.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v19.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v24, v9
-; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v10.l, v10.h
-; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v24.h
-; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v70.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v68.l
-; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v11.h, v12.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v66.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v24, v10
-; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v53.h
-; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v11.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v24.h
-; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v52.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.l, 8, v65.l
-; GFX11-TRUE16-NEXT: v_and_b16 v13.h, 0xff, v13.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v18.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v24, v11
-; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v12.l, v12.h
-; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v24.h
-; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v85.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.l, 8, v64.l
-; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v13.h, v14.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v54.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v24, v12
-; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v67.h
-; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v13.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v24.h
-; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v69.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v51.l
-; GFX11-TRUE16-NEXT: v_and_b16 v15.h, 0xff, v15.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v17.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v24, v13
-; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v14.l, v14.h
-; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v24.h
-; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v87.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v50.l
-; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v15.h, v16.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v49.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v24, v14
-; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v86.h
-; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v15.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v24.h
-; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v83.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v48.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v24, v15
-; GFX11-TRUE16-NEXT: v_or_b16 v24.l, v16.l, v16.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v53.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v131.l
+; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v17.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v51.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.h, 0
+; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v18.h
+; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v1.l, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v129.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.l, v2.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v31.h
+; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v39.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v130.l
+; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v19.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v50.l
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v31, v1
+; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v20.h
+; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v2.l, v2.h
+; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v3.l, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v31.h
+; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v55.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v128.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v118.l
+; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v21.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v31, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v49.l
+; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v3.l, v3.h
+; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v4.l, v4.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v31.h
+; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v54.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v119.l
+; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v22.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v115.l
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v31, v3
+; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v23.h
+; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v4.l, v4.h
+; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v5.l, v5.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v31.h
+; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v65.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v117.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v48.l
+; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v24.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v31, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v101.l
+; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v5.l, v5.h
+; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v6.l, v6.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v31.h
+; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v64.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v116.l
+; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v25.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v35.l
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v31, v5
+; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v26.h
+; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v6.l, v6.h
+; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.l, v7.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v31.h
+; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v68.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v113.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v98.l
+; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v27.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v31, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v34.l
+; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v7.l, v7.h
+; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v8.l, v8.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v31.h
+; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v66.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v102.l
+; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v28.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v85.l
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v31, v7
+; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v32.h
+; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v8.l, v8.h
+; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v9.l, v9.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v31.h
+; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v83.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v100.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v30.l
+; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v33.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v31, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v80.l
+; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v9.l, v9.h
+; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v10.l, v10.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v31.h
+; GFX11-TRUE16-NEXT: v_and_b16 v10.l, 0xff, v67.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v99.l
+; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v37.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v29.l
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v31, v9
+; GFX11-TRUE16-NEXT: v_and_b16 v17.l, 0xff, v38.h
+; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v10.l, v10.h
+; GFX11-TRUE16-NEXT: v_or_b16 v10.h, v11.l, v11.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v31.h
+; GFX11-TRUE16-NEXT: v_and_b16 v11.l, 0xff, v97.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v87.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v69.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, v31, v10
+; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v11.l, v11.h
+; GFX11-TRUE16-NEXT: v_or_b16 v11.h, v12.l, v12.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v31.h
+; GFX11-TRUE16-NEXT: v_and_b16 v12.l, 0xff, v82.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v86.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, v31, v11
+; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v12.l, v12.h
+; GFX11-TRUE16-NEXT: v_or_b16 v12.h, v13.l, v13.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v31.h
+; GFX11-TRUE16-NEXT: v_and_b16 v13.l, 0xff, v112.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v84.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, v31, v12
+; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v13.l, v13.h
+; GFX11-TRUE16-NEXT: v_or_b16 v13.h, v14.l, v14.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v31.h
+; GFX11-TRUE16-NEXT: v_and_b16 v14.l, 0xff, v96.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v14.h, 8, v81.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, v31, v13
+; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v14.l, v14.h
+; GFX11-TRUE16-NEXT: v_or_b16 v14.h, v15.l, v15.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v31.h
+; GFX11-TRUE16-NEXT: v_and_b16 v15.l, 0xff, v114.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v15.h, 8, v71.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, v31, v14
+; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v15.l, v15.h
+; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v16.l, v16.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v31.h
+; GFX11-TRUE16-NEXT: v_and_b16 v16.l, 0xff, v103.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v70.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, v31, v15
+; GFX11-TRUE16-NEXT: v_or_b16 v31.l, v16.l, v16.h
; GFX11-TRUE16-NEXT: v_or_b16 v16.h, v17.l, v17.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v24.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v24, v16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v31.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, v31, v16
; GFX11-TRUE16-NEXT: s_clause 0x3
; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[1:4], off
; GFX11-TRUE16-NEXT: scratch_store_b128 v0, v[5:8], off offset:16
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
index 4c485768bcbbf..ab1f8606cffd7 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
@@ -2153,56 +2153,56 @@ define i64 @bitcast_v4bf16_to_i64(<4 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v4bf16_to_i64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v2
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v3
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v9, v11 :: v_dual_and_b32 v0, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v1, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v3, v4
-; GFX11-TRUE16-NEXT: .LBB22_2: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.h
+; GFX11-TRUE16-NEXT: .LBB22_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4bf16_to_i64:
@@ -5288,56 +5288,56 @@ define double @bitcast_v4bf16_to_f64(<4 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v4bf16_to_f64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v2
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v3
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v9, v11 :: v_dual_and_b32 v0, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v1, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v3, v4
-; GFX11-TRUE16-NEXT: .LBB46_2: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.h
+; GFX11-TRUE16-NEXT: .LBB46_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4bf16_to_f64:
@@ -8135,56 +8135,56 @@ define <2 x i32> @bitcast_v4bf16_to_v2i32(<4 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v4bf16_to_v2i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v2
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB66_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB66_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v3
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v9, v11 :: v_dual_and_b32 v0, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v1, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v3, v4
-; GFX11-TRUE16-NEXT: .LBB66_2: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.h
+; GFX11-TRUE16-NEXT: .LBB66_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4bf16_to_v2i32:
@@ -10655,56 +10655,56 @@ define <2 x float> @bitcast_v4bf16_to_v2f32(<4 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v4bf16_to_v2f32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v2
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v3
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v9, v11 :: v_dual_and_b32 v0, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v1, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v3, v4
-; GFX11-TRUE16-NEXT: .LBB82_2: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.h
+; GFX11-TRUE16-NEXT: .LBB82_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4bf16_to_v2f32:
@@ -14628,55 +14628,58 @@ define <4 x half> @bitcast_v4bf16_to_v4f16(<4 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v4bf16_to_v4f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v2
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB102_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB102_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v3
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v9, v11 :: v_dual_add_f32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.h
; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v3, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v0
-; GFX11-TRUE16-NEXT: .LBB102_2: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.h
+; GFX11-TRUE16-NEXT: .LBB102_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4bf16_to_v4f16:
@@ -16248,78 +16251,78 @@ define <8 x i8> @bitcast_v4bf16_to_v8i8(<4 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v4bf16_to_v8i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v2
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[10:11], 24, v[8:9]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v9.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[6:7], 24, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v0.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v1.h
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX11-TRUE16-NEXT: .LBB108_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v8.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v0 :: v_dual_add_f32 v0, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v9.l
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v2 :: v_dual_add_f32 v2, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v4, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v11, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v8, 0xffff, v2, v1
-; GFX11-TRUE16-NEXT: v_bfi_b32 v9, 0xffff, v3, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[10:11], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v8, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v1, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v12, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[6:7], 24, v[8:9]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
; GFX11-TRUE16-NEXT: .LBB108_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v8.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v9.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4bf16_to_v8i8:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
index 879e8520d8e18..4aded5da3668a 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
@@ -2075,66 +2075,75 @@ define <3 x i32> @bitcast_v6bf16_to_v3i32(<6 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v6bf16_to_v3i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v3
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_vgpr4_vgpr5
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB10_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB10_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v2, 0x40c00000, v2
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v3, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v0, v1, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_cndmask_b32 v5, v7, v9
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v1, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v0, v8 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v7.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v11
; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v6 :: v_dual_cndmask_b32 v0, v0, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v1, v10, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v9, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v12, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v3, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v10, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v7.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v3, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v6, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v4, v5
-; GFX11-TRUE16-NEXT: .LBB10_2: ; %end
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v1, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v7.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v10.h
+; GFX11-TRUE16-NEXT: .LBB10_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v3 :: v_dual_mov_b32 v1, v4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v6bf16_to_v3i32:
@@ -5213,66 +5222,75 @@ define <3 x float> @bitcast_v6bf16_to_v3f32(<6 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v6bf16_to_v3f32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v3
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_vgpr4_vgpr5
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB26_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB26_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v2, 0x40c00000, v2
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v3, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v0, v1, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_cndmask_b32 v5, v7, v9
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v1, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v0, v8 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v7.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v11
; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v6 :: v_dual_cndmask_b32 v0, v0, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v1, v10, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v9, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v12, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v3, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v10, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v7.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v3, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v6, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v4, v5
-; GFX11-TRUE16-NEXT: .LBB26_2: ; %end
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v1, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v7.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v10.h
+; GFX11-TRUE16-NEXT: .LBB26_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v3 :: v_dual_mov_b32 v1, v4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v6bf16_to_v3f32:
@@ -7734,111 +7752,109 @@ define <12 x i8> @bitcast_v6bf16_to_v12i8(<6 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v6bf16_to_v12i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, v2 :: v_dual_mov_b32 v13, v1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v12, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v3
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB38_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v10
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v13
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v13
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[16:17], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[14:15], 24, v[12:13]
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v13.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v10.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[14:15], 24, v[2:3]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[12:13], 24, v[0:1]
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v0.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v1.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v2.h
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2
; GFX11-TRUE16-NEXT: .LBB38_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB38_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v13.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_add_f32 v3, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v12.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v2, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v17, 0x7fc07fc0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v4, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v10.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v10
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v11, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v11, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v4, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v10, v7, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v13, v0, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v13, v14, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v12, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v13, 0xffff, v7, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v12, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v11, v0, v8 :: v_dual_add_f32 v0, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v10, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v3, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v13
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v12, 0xffff, v2, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0x7fc07fc0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[14:15], 24, v[12:13]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v12
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v5, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v13
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[16:17], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v10
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v4.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v10, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v2, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v3.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v8.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[12:13], 24, v[10:11]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[14:15], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
; GFX11-TRUE16-NEXT: .LBB38_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v12.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v12.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v11.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v16.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v14.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v6bf16_to_v12i8:
@@ -11413,71 +11429,74 @@ define <6 x half> @bitcast_v6bf16_to_v6f16(<6 x bfloat> %a, i32 %b) {
; GFX11-TRUE16-LABEL: bitcast_v6bf16_to_v6f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v3
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_vgpr4_vgpr5_vgpr6
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB48_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB48_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v6, 0x40c00000, v4
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v4
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v1, v7, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_cndmask_b32 v3, v1, v7
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_cndmask_b32 v1, v8, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v2, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v0, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v9, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v10, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v9, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v7, v12 :: v_dual_add_f32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v6, v8 :: v_dual_add_f32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v11, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v11, v5, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v6, v0
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v7, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v4, v5
-; GFX11-TRUE16-NEXT: .LBB48_2: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v7.h
+; GFX11-TRUE16-NEXT: .LBB48_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v3 :: v_dual_mov_b32 v1, v4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v6bf16_to_v6f16:
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index 0fccdba729132..97df2a0dbd44b 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -328,7 +328,7 @@ define amdgpu_kernel void @add_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1232-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX1232-NEXT: s_endpgm
entry:
- %old = atomicrmw add ptr addrspace(1) %inout, i32 5 syncscope("agent") acq_rel, !amdgpu.no.fine.grained.memory !0
+ %old = atomicrmw add ptr addrspace(1) %inout, i32 5 syncscope("agent") acq_rel
store i32 %old, ptr addrspace(1) %out
ret void
}
@@ -655,7 +655,7 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1232-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX1232-NEXT: s_endpgm
entry:
- %old = atomicrmw add ptr addrspace(1) %inout, i32 %additive syncscope("agent") acq_rel, !amdgpu.no.fine.grained.memory !0
+ %old = atomicrmw add ptr addrspace(1) %inout, i32 %additive syncscope("agent") acq_rel
store i32 %old, ptr addrspace(1) %out
ret void
}
@@ -1565,7 +1565,7 @@ define amdgpu_kernel void @add_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1232_DPP-NEXT: s_endpgm
entry:
%lane = call i32 @llvm.amdgcn.workitem.id.x()
- %old = atomicrmw add ptr addrspace(1) %inout, i32 %lane syncscope("agent") acq_rel, !amdgpu.no.fine.grained.memory !0
+ %old = atomicrmw add ptr addrspace(1) %inout, i32 %lane syncscope("agent") acq_rel
store i32 %old, ptr addrspace(1) %out
ret void
}
@@ -1899,7 +1899,7 @@ define amdgpu_kernel void @add_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1232-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null
; GFX1232-NEXT: s_endpgm
entry:
- %old = atomicrmw add ptr addrspace(1) %inout, i64 5 syncscope("agent") acq_rel, !amdgpu.no.fine.grained.memory !0
+ %old = atomicrmw add ptr addrspace(1) %inout, i64 5 syncscope("agent") acq_rel
store i64 %old, ptr addrspace(1) %out
ret void
}
@@ -2284,7 +2284,7 @@ define amdgpu_kernel void @add_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1232-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null
; GFX1232-NEXT: s_endpgm
entry:
- %old = atomicrmw add ptr addrspace(1) %inout, i64 %additive syncscope("agent") acq_rel, !amdgpu.no.fine.grained.memory !0
+ %old = atomicrmw add ptr addrspace(1) %inout, i64 %additive syncscope("agent") acq_rel
store i64 %old, ptr addrspace(1) %out
ret void
}
@@ -3545,7 +3545,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
entry:
%lane = call i32 @llvm.amdgcn.workitem.id.x()
%zext = zext i32 %lane to i64
- %old = atomicrmw add ptr addrspace(1) %inout, i64 %zext syncscope("agent") acq_rel, !amdgpu.no.fine.grained.memory !0
+ %old = atomicrmw add ptr addrspace(1) %inout, i64 %zext syncscope("agent") acq_rel
store i64 %old, ptr addrspace(1) %out
ret void
}
@@ -3556,30 +3556,47 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: ; implicit-def: $vgpr1
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX7LESS-NEXT: s_cbranch_execz .LBB6_2
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s7, v0
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: ; implicit-def: $vgpr0
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX7LESS-NEXT: s_cbranch_execz .LBB6_4
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_mov_b32 s11, 0xf000
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX7LESS-NEXT: s_mul_i32 s6, s6, 5
-; GFX7LESS-NEXT: s_mov_b32 s10, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mov_b32 s8, s2
-; GFX7LESS-NEXT: s_mov_b32 s9, s3
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
-; GFX7LESS-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc
+; GFX7LESS-NEXT: s_load_dword s4, s[2:3], 0x0
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
+; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_mul_i32 s12, s12, 5
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s4, s2
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
+; GFX7LESS-NEXT: .LBB6_2: ; %atomicrmw.start
+; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: v_subrev_i32_e32 v3, vcc, s12, v4
+; GFX7LESS-NEXT: s_waitcnt expcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v4
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: buffer_wbinvl1
-; GFX7LESS-NEXT: .LBB6_2:
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX7LESS-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX7LESS-NEXT: s_cbranch_execnz .LBB6_2
+; GFX7LESS-NEXT: ; %bb.3: ; %Flow
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX7LESS-NEXT: .LBB6_4: ; %Flow3
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v1
-; GFX7LESS-NEXT: v_mul_u32_u24_e32 v0, 5, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v0
+; GFX7LESS-NEXT: s_waitcnt expcnt(0)
+; GFX7LESS-NEXT: v_mul_u32_u24_e32 v0, 5, v2
; GFX7LESS-NEXT: v_sub_i32_e32 v0, vcc, s4, v0
; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7LESS-NEXT: s_endpgm
@@ -3589,27 +3606,42 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_mov_b64 s[6:7], exec
; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX8-NEXT: ; implicit-def: $vgpr1
-; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX8-NEXT: s_cbranch_execz .LBB6_2
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX8-NEXT: ; implicit-def: $vgpr0
+; GFX8-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX8-NEXT: s_cbranch_execz .LBB6_4
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mov_b32 s8, s2
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX8-NEXT: s_mul_i32 s2, s2, 5
-; GFX8-NEXT: s_mov_b32 s11, 0xf000
-; GFX8-NEXT: s_mov_b32 s10, -1
-; GFX8-NEXT: s_mov_b32 s9, s3
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc
+; GFX8-NEXT: s_load_dword s4, s[2:3], 0x0
+; GFX8-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
+; GFX8-NEXT: s_mov_b64 s[10:11], 0
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mul_i32 s12, s12, 5
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v0, s4
+; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_mov_b32 s4, s2
+; GFX8-NEXT: s_mov_b32 s5, s3
+; GFX8-NEXT: .LBB6_2: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_mov_b32_e32 v4, v0
+; GFX8-NEXT: v_subrev_u32_e32 v3, vcc, s12, v4
+; GFX8-NEXT: v_mov_b32_e32 v0, v3
+; GFX8-NEXT: v_mov_b32_e32 v1, v4
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: .LBB6_2:
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT: v_readfirstlane_b32 s4, v1
-; GFX8-NEXT: v_mul_u32_u24_e32 v0, 5, v0
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX8-NEXT: s_cbranch_execnz .LBB6_2
+; GFX8-NEXT: ; %bb.3: ; %Flow
+; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX8-NEXT: .LBB6_4: ; %Flow3
+; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX8-NEXT: v_readfirstlane_b32 s4, v0
+; GFX8-NEXT: v_mul_u32_u24_e32 v0, 5, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_mov_b32 s2, -1
@@ -3622,27 +3654,42 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[6:7], exec
; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: ; implicit-def: $vgpr1
-; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX9-NEXT: s_cbranch_execz .LBB6_2
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: ; implicit-def: $vgpr0
+; GFX9-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX9-NEXT: s_cbranch_execz .LBB6_4
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s8, s2
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX9-NEXT: s_mul_i32 s2, s2, 5
-; GFX9-NEXT: s_mov_b32 s11, 0xf000
-; GFX9-NEXT: s_mov_b32 s10, -1
-; GFX9-NEXT: s_mov_b32 s9, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc
+; GFX9-NEXT: s_load_dword s4, s[2:3], 0x0
+; GFX9-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
+; GFX9-NEXT: s_mov_b64 s[10:11], 0
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mul_i32 s12, s12, 5
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_mov_b32 s4, s2
+; GFX9-NEXT: s_mov_b32 s5, s3
+; GFX9-NEXT: .LBB6_2: ; %atomicrmw.start
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: v_mov_b32_e32 v4, v0
+; GFX9-NEXT: v_subrev_u32_e32 v3, s12, v4
+; GFX9-NEXT: v_mov_b32_e32 v0, v3
+; GFX9-NEXT: v_mov_b32_e32 v1, v4
+; GFX9-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: .LBB6_2:
-; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-NEXT: v_readfirstlane_b32 s4, v1
-; GFX9-NEXT: v_mul_u32_u24_e32 v0, 5, v0
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX9-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX9-NEXT: s_cbranch_execnz .LBB6_2
+; GFX9-NEXT: ; %bb.3: ; %Flow
+; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX9-NEXT: .LBB6_4: ; %Flow3
+; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX9-NEXT: v_readfirstlane_b32 s4, v0
+; GFX9-NEXT: v_mul_u32_u24_e32 v0, 5, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_mov_b32 s3, 0xf000
; GFX9-NEXT: s_mov_b32 s2, -1
@@ -3654,31 +3701,45 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1064: ; %bb.0: ; %entry
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064-NEXT: s_mov_b64 s[6:7], exec
-; GFX1064-NEXT: ; implicit-def: $vgpr1
; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX1064-NEXT: s_cbranch_execz .LBB6_2
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1064-NEXT: ; implicit-def: $vgpr0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX1064-NEXT: s_cbranch_execz .LBB6_4
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1064-NEXT: s_mul_i32 s6, s6, 5
-; GFX1064-NEXT: s_mov_b32 s10, -1
-; GFX1064-NEXT: v_mov_b32_e32 v1, s6
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_mov_b32 s8, s2
-; GFX1064-NEXT: s_mov_b32 s9, s3
-; GFX1064-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc
+; GFX1064-NEXT: s_load_dword s4, s[2:3], 0x0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
+; GFX1064-NEXT: s_mov_b64 s[10:11], 0
+; GFX1064-NEXT: s_mul_i32 s12, s12, 5
+; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT: s_mov_b32 s6, -1
+; GFX1064-NEXT: s_mov_b32 s5, s3
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: v_mov_b32_e32 v0, s4
+; GFX1064-NEXT: s_mov_b32 s4, s2
+; GFX1064-NEXT: .LBB6_2: ; %atomicrmw.start
+; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: v_mov_b32_e32 v4, v0
+; GFX1064-NEXT: v_subrev_nc_u32_e32 v3, s12, v4
+; GFX1064-NEXT: v_mov_b32_e32 v0, v3
+; GFX1064-NEXT: v_mov_b32_e32 v1, v4
+; GFX1064-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: buffer_gl1_inv
; GFX1064-NEXT: buffer_gl0_inv
-; GFX1064-NEXT: .LBB6_2:
-; GFX1064-NEXT: s_waitcnt_depctr 0xffe3
-; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX1064-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX1064-NEXT: s_cbranch_execnz .LBB6_2
+; GFX1064-NEXT: ; %bb.3: ; %Flow
+; GFX1064-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1064-NEXT: .LBB6_4: ; %Flow3
+; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_readfirstlane_b32 s2, v1
-; GFX1064-NEXT: v_mul_u32_u24_e32 v0, 5, v0
+; GFX1064-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1064-NEXT: v_mul_u32_u24_e32 v0, 5, v2
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
; GFX1064-NEXT: v_sub_nc_u32_e32 v0, s2, v0
; GFX1064-NEXT: s_mov_b32 s2, -1
@@ -3689,30 +3750,44 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1032: ; %bb.0: ; %entry
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032-NEXT: s_mov_b32 s6, exec_lo
-; GFX1032-NEXT: ; implicit-def: $vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo
-; GFX1032-NEXT: s_cbranch_execz .LBB6_2
+; GFX1032-NEXT: s_mov_b32 s9, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
+; GFX1032-NEXT: ; implicit-def: $vgpr0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s8, vcc_lo
+; GFX1032-NEXT: s_cbranch_execz .LBB6_4
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1032-NEXT: s_mul_i32 s5, s5, 5
-; GFX1032-NEXT: s_mov_b32 s10, -1
-; GFX1032-NEXT: v_mov_b32_e32 v1, s5
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_mov_b32 s8, s2
-; GFX1032-NEXT: s_mov_b32 s9, s3
-; GFX1032-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc
+; GFX1032-NEXT: s_load_dword s4, s[2:3], 0x0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s10, s6
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: s_mul_i32 s10, s10, 5
+; GFX1032-NEXT: s_mov_b32 s6, -1
+; GFX1032-NEXT: s_mov_b32 s5, s3
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v0, s4
+; GFX1032-NEXT: s_mov_b32 s4, s2
+; GFX1032-NEXT: .LBB6_2: ; %atomicrmw.start
+; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: v_mov_b32_e32 v4, v0
+; GFX1032-NEXT: v_subrev_nc_u32_e32 v3, s10, v4
+; GFX1032-NEXT: v_mov_b32_e32 v0, v3
+; GFX1032-NEXT: v_mov_b32_e32 v1, v4
+; GFX1032-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: buffer_gl1_inv
; GFX1032-NEXT: buffer_gl0_inv
-; GFX1032-NEXT: .LBB6_2:
-; GFX1032-NEXT: s_waitcnt_depctr 0xffe3
-; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX1032-NEXT: s_or_b32 s9, vcc_lo, s9
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
+; GFX1032-NEXT: s_cbranch_execnz .LBB6_2
+; GFX1032-NEXT: ; %bb.3: ; %Flow
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1032-NEXT: .LBB6_4: ; %Flow3
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_readfirstlane_b32 s2, v1
-; GFX1032-NEXT: v_mul_u32_u24_e32 v0, 5, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032-NEXT: v_mul_u32_u24_e32 v0, 5, v2
; GFX1032-NEXT: s_mov_b32 s3, 0x31016000
; GFX1032-NEXT: v_sub_nc_u32_e32 v0, s2, v0
; GFX1032-NEXT: s_mov_b32 s2, -1
@@ -3723,33 +3798,51 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1164: ; %bb.0: ; %entry
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164-NEXT: s_mov_b64 s[6:7], exec
-; GFX1164-NEXT: s_mov_b64 s[4:5], exec
+; GFX1164-NEXT: s_mov_b64 s[8:9], exec
; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_cbranch_execz .LBB6_2
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1164-NEXT: ; implicit-def: $vgpr0
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-NEXT: s_cbranch_execz .LBB6_4
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1164-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1164-NEXT: s_mul_i32 s6, s6, 5
-; GFX1164-NEXT: s_mov_b32 s10, -1
-; GFX1164-NEXT: v_mov_b32_e32 v1, s6
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_mov_b32 s8, s2
-; GFX1164-NEXT: s_mov_b32 s9, s3
-; GFX1164-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc
+; GFX1164-NEXT: s_load_b32 s4, s[2:3], 0x0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s12, s[6:7]
+; GFX1164-NEXT: s_mov_b64 s[10:11], 0
+; GFX1164-NEXT: s_mul_i32 s12, s12, 5
+; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-NEXT: s_mov_b32 s6, -1
+; GFX1164-NEXT: s_mov_b32 s5, s3
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: v_mov_b32_e32 v0, s4
+; GFX1164-NEXT: s_mov_b32 s4, s2
+; GFX1164-NEXT: .LBB6_2: ; %atomicrmw.start
+; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_mov_b32_e32 v4, v0
+; GFX1164-NEXT: v_subrev_nc_u32_e32 v3, s12, v4
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mov_b32_e32 v0, v3
+; GFX1164-NEXT: v_mov_b32_e32 v1, v4
+; GFX1164-NEXT: buffer_atomic_cmpswap_b32 v[0:1], off, s[4:7], 0 glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: buffer_gl1_inv
; GFX1164-NEXT: buffer_gl0_inv
-; GFX1164-NEXT: .LBB6_2:
-; GFX1164-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX1164-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1164-NEXT: s_cbranch_execnz .LBB6_2
+; GFX1164-NEXT: ; %bb.3: ; %Flow
+; GFX1164-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1164-NEXT: .LBB6_4: ; %Flow3
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_readfirstlane_b32 s2, v1
-; GFX1164-NEXT: v_mul_u32_u24_e32 v0, 5, v0
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1164-NEXT: v_mul_u32_u24_e32 v0, 5, v2
; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164-NEXT: v_sub_nc_u32_e32 v0, s2, v0
; GFX1164-NEXT: s_mov_b32 s2, -1
; GFX1164-NEXT: buffer_store_b32 v0, off, s[0:3], 0
@@ -3759,32 +3852,49 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1132: ; %bb.0: ; %entry
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132-NEXT: s_mov_b32 s6, exec_lo
-; GFX1132-NEXT: s_mov_b32 s4, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1132-NEXT: ; implicit-def: $vgpr1
+; GFX1132-NEXT: s_mov_b32 s9, 0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
+; GFX1132-NEXT: s_mov_b32 s8, exec_lo
+; GFX1132-NEXT: ; implicit-def: $vgpr0
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_cbranch_execz .LBB6_2
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-NEXT: s_cbranch_execz .LBB6_4
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1132-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1132-NEXT: s_mul_i32 s5, s5, 5
-; GFX1132-NEXT: s_mov_b32 s10, -1
-; GFX1132-NEXT: v_mov_b32_e32 v1, s5
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_mov_b32 s8, s2
-; GFX1132-NEXT: s_mov_b32 s9, s3
-; GFX1132-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc
+; GFX1132-NEXT: s_load_b32 s4, s[2:3], 0x0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s10, s6
+; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-NEXT: s_mul_i32 s10, s10, 5
+; GFX1132-NEXT: s_mov_b32 s6, -1
+; GFX1132-NEXT: s_mov_b32 s5, s3
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: v_mov_b32_e32 v0, s4
+; GFX1132-NEXT: s_mov_b32 s4, s2
+; GFX1132-NEXT: .LBB6_2: ; %atomicrmw.start
+; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mov_b32_e32 v4, v0
+; GFX1132-NEXT: v_subrev_nc_u32_e32 v3, s10, v4
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_dual_mov_b32 v0, v3 :: v_dual_mov_b32 v1, v4
+; GFX1132-NEXT: buffer_atomic_cmpswap_b32 v[0:1], off, s[4:7], 0 glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: buffer_gl1_inv
; GFX1132-NEXT: buffer_gl0_inv
-; GFX1132-NEXT: .LBB6_2:
-; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX1132-NEXT: s_or_b32 s9, vcc_lo, s9
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s9
+; GFX1132-NEXT: s_cbranch_execnz .LBB6_2
+; GFX1132-NEXT: ; %bb.3: ; %Flow
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1132-NEXT: .LBB6_4: ; %Flow3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_readfirstlane_b32 s2, v1
-; GFX1132-NEXT: v_mul_u32_u24_e32 v0, 5, v0
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132-NEXT: v_mul_u32_u24_e32 v0, 5, v2
; GFX1132-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_sub_nc_u32_e32 v0, s2, v0
; GFX1132-NEXT: s_mov_b32 s2, -1
; GFX1132-NEXT: buffer_store_b32 v0, off, s[0:3], 0
@@ -3861,7 +3971,7 @@ define amdgpu_kernel void @sub_i32_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1232-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX1232-NEXT: s_endpgm
entry:
- %old = atomicrmw sub ptr addrspace(1) %inout, i32 5 syncscope("agent") acq_rel, !amdgpu.no.fine.grained.memory !0
+ %old = atomicrmw sub ptr addrspace(1) %inout, i32 5 syncscope("agent") acq_rel
store i32 %old, ptr addrspace(1) %out
ret void
}
@@ -3871,32 +3981,49 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS: ; %bb.0: ; %entry
; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_load_dword s8, s[4:5], 0xd
+; GFX7LESS-NEXT: s_load_dword s12, s[4:5], 0xd
; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS-NEXT: ; implicit-def: $vgpr1
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX7LESS-NEXT: s_cbranch_execz .LBB7_2
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s7, v0
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: ; implicit-def: $vgpr0
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX7LESS-NEXT: s_cbranch_execz .LBB7_4
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xf000
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mul_i32 s6, s8, s6
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
-; GFX7LESS-NEXT: s_mov_b32 s12, s2
-; GFX7LESS-NEXT: s_mov_b32 s13, s3
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
-; GFX7LESS-NEXT: buffer_atomic_sub v1, off, s[12:15], 0 glc
+; GFX7LESS-NEXT: s_load_dword s4, s[2:3], 0x0
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s5, s[6:7]
+; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_mul_i32 s13, s12, s5
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s4, s2
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
+; GFX7LESS-NEXT: .LBB7_2: ; %atomicrmw.start
+; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS-NEXT: v_subrev_i32_e32 v3, vcc, s13, v4
+; GFX7LESS-NEXT: s_waitcnt expcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v4
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: buffer_wbinvl1
-; GFX7LESS-NEXT: .LBB7_2:
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX7LESS-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX7LESS-NEXT: s_cbranch_execnz .LBB7_2
+; GFX7LESS-NEXT: ; %bb.3: ; %Flow
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX7LESS-NEXT: .LBB7_4: ; %Flow3
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v1
-; GFX7LESS-NEXT: v_mul_lo_u32 v0, s8, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v0
+; GFX7LESS-NEXT: s_waitcnt expcnt(0)
+; GFX7LESS-NEXT: v_mul_lo_u32 v0, s12, v2
; GFX7LESS-NEXT: v_sub_i32_e32 v0, vcc, s4, v0
; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7LESS-NEXT: s_endpgm
@@ -3904,68 +4031,98 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX8-LABEL: sub_i32_uniform:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_load_dword s8, s[4:5], 0x34
+; GFX8-NEXT: s_load_dword s12, s[4:5], 0x34
; GFX8-NEXT: s_mov_b64 s[6:7], exec
; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX8-NEXT: ; implicit-def: $vgpr1
-; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX8-NEXT: s_cbranch_execz .LBB7_2
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX8-NEXT: ; implicit-def: $vgpr0
+; GFX8-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX8-NEXT: s_cbranch_execz .LBB7_4
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mov_b32 s12, s2
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX8-NEXT: s_mul_i32 s2, s8, s2
-; GFX8-NEXT: s_mov_b32 s15, 0xf000
-; GFX8-NEXT: s_mov_b32 s14, -1
-; GFX8-NEXT: s_mov_b32 s13, s3
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: buffer_atomic_sub v1, off, s[12:15], 0 glc
+; GFX8-NEXT: s_load_dword s4, s[2:3], 0x0
+; GFX8-NEXT: s_bcnt1_i32_b64 s5, s[6:7]
+; GFX8-NEXT: s_mov_b64 s[10:11], 0
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mul_i32 s13, s12, s5
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v0, s4
+; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_mov_b32 s4, s2
+; GFX8-NEXT: s_mov_b32 s5, s3
+; GFX8-NEXT: .LBB7_2: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_mov_b32_e32 v4, v0
+; GFX8-NEXT: v_subrev_u32_e32 v3, vcc, s13, v4
+; GFX8-NEXT: v_mov_b32_e32 v0, v3
+; GFX8-NEXT: v_mov_b32_e32 v1, v4
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: .LBB7_2:
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX8-NEXT: s_cbranch_execnz .LBB7_2
+; GFX8-NEXT: ; %bb.3: ; %Flow
+; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX8-NEXT: .LBB7_4: ; %Flow3
+; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u32 v0, s8, v0
-; GFX8-NEXT: v_readfirstlane_b32 s4, v1
+; GFX8-NEXT: v_mul_lo_u32 v1, s12, v2
+; GFX8-NEXT: v_readfirstlane_b32 s4, v0
; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_mov_b32 s2, -1
-; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s4, v0
+; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s4, v1
; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: sub_i32_uniform:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s8, s[4:5], 0x34
+; GFX9-NEXT: s_load_dword s12, s[4:5], 0x34
; GFX9-NEXT: s_mov_b64 s[6:7], exec
; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: ; implicit-def: $vgpr1
-; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX9-NEXT: s_cbranch_execz .LBB7_2
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: ; implicit-def: $vgpr0
+; GFX9-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX9-NEXT: s_cbranch_execz .LBB7_4
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s12, s2
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX9-NEXT: s_mul_i32 s2, s8, s2
-; GFX9-NEXT: s_mov_b32 s15, 0xf000
-; GFX9-NEXT: s_mov_b32 s14, -1
-; GFX9-NEXT: s_mov_b32 s13, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: buffer_atomic_sub v1, off, s[12:15], 0 glc
+; GFX9-NEXT: s_load_dword s4, s[2:3], 0x0
+; GFX9-NEXT: s_bcnt1_i32_b64 s5, s[6:7]
+; GFX9-NEXT: s_mov_b64 s[10:11], 0
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mul_i32 s13, s12, s5
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_mov_b32 s4, s2
+; GFX9-NEXT: s_mov_b32 s5, s3
+; GFX9-NEXT: .LBB7_2: ; %atomicrmw.start
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: v_mov_b32_e32 v4, v0
+; GFX9-NEXT: v_subrev_u32_e32 v3, s13, v4
+; GFX9-NEXT: v_mov_b32_e32 v0, v3
+; GFX9-NEXT: v_mov_b32_e32 v1, v4
+; GFX9-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: .LBB7_2:
-; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX9-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX9-NEXT: s_cbranch_execnz .LBB7_2
+; GFX9-NEXT: ; %bb.3: ; %Flow
+; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX9-NEXT: .LBB7_4: ; %Flow3
+; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mul_lo_u32 v0, s8, v0
-; GFX9-NEXT: v_readfirstlane_b32 s4, v1
+; GFX9-NEXT: v_mul_lo_u32 v1, s12, v2
+; GFX9-NEXT: v_readfirstlane_b32 s4, v0
; GFX9-NEXT: s_mov_b32 s3, 0xf000
; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: v_sub_u32_e32 v0, s4, v0
+; GFX9-NEXT: v_sub_u32_e32 v0, s4, v1
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX9-NEXT: s_endpgm
;
@@ -3973,35 +4130,49 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1064: ; %bb.0: ; %entry
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1064-NEXT: s_load_dword s8, s[4:5], 0x34
+; GFX1064-NEXT: s_load_dword s12, s[4:5], 0x34
; GFX1064-NEXT: s_mov_b64 s[6:7], exec
-; GFX1064-NEXT: ; implicit-def: $vgpr1
; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX1064-NEXT: s_cbranch_execz .LBB7_2
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1064-NEXT: ; implicit-def: $vgpr0
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX1064-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX1064-NEXT: s_cbranch_execz .LBB7_4
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1064-NEXT: s_mov_b32 s15, 0x31016000
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_mul_i32 s6, s8, s6
-; GFX1064-NEXT: s_mov_b32 s14, -1
-; GFX1064-NEXT: v_mov_b32_e32 v1, s6
-; GFX1064-NEXT: s_mov_b32 s12, s2
-; GFX1064-NEXT: s_mov_b32 s13, s3
-; GFX1064-NEXT: buffer_atomic_sub v1, off, s[12:15], 0 glc
+; GFX1064-NEXT: s_load_dword s4, s[2:3], 0x0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s5, s[6:7]
+; GFX1064-NEXT: s_mov_b64 s[10:11], 0
+; GFX1064-NEXT: s_mul_i32 s13, s12, s5
+; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT: s_mov_b32 s6, -1
+; GFX1064-NEXT: s_mov_b32 s5, s3
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: v_mov_b32_e32 v0, s4
+; GFX1064-NEXT: s_mov_b32 s4, s2
+; GFX1064-NEXT: .LBB7_2: ; %atomicrmw.start
+; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: v_mov_b32_e32 v4, v0
+; GFX1064-NEXT: v_subrev_nc_u32_e32 v3, s13, v4
+; GFX1064-NEXT: v_mov_b32_e32 v0, v3
+; GFX1064-NEXT: v_mov_b32_e32 v1, v4
+; GFX1064-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: buffer_gl1_inv
; GFX1064-NEXT: buffer_gl0_inv
-; GFX1064-NEXT: .LBB7_2:
-; GFX1064-NEXT: s_waitcnt_depctr 0xffe3
-; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX1064-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX1064-NEXT: s_cbranch_execnz .LBB7_2
+; GFX1064-NEXT: ; %bb.3: ; %Flow
+; GFX1064-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1064-NEXT: .LBB7_4: ; %Flow3
+; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mul_lo_u32 v0, s8, v0
-; GFX1064-NEXT: v_readfirstlane_b32 s2, v1
+; GFX1064-NEXT: v_mul_lo_u32 v1, s12, v2
+; GFX1064-NEXT: v_readfirstlane_b32 s2, v0
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1064-NEXT: v_sub_nc_u32_e32 v0, s2, v0
+; GFX1064-NEXT: v_sub_nc_u32_e32 v0, s2, v1
; GFX1064-NEXT: s_mov_b32 s2, -1
; GFX1064-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX1064-NEXT: s_endpgm
@@ -4010,34 +4181,48 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1032: ; %bb.0: ; %entry
; GFX1032-NEXT: s_clause 0x1
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1032-NEXT: s_load_dword s6, s[4:5], 0x34
-; GFX1032-NEXT: s_mov_b32 s7, exec_lo
-; GFX1032-NEXT: ; implicit-def: $vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s7, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo
-; GFX1032-NEXT: s_cbranch_execz .LBB7_2
+; GFX1032-NEXT: s_load_dword s8, s[4:5], 0x34
+; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: s_mov_b32 s10, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
+; GFX1032-NEXT: ; implicit-def: $vgpr0
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX1032-NEXT: s_and_saveexec_b32 s9, vcc_lo
+; GFX1032-NEXT: s_cbranch_execz .LBB7_4
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s7
-; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_mul_i32 s5, s6, s5
-; GFX1032-NEXT: s_mov_b32 s10, -1
-; GFX1032-NEXT: v_mov_b32_e32 v1, s5
-; GFX1032-NEXT: s_mov_b32 s8, s2
-; GFX1032-NEXT: s_mov_b32 s9, s3
-; GFX1032-NEXT: buffer_atomic_sub v1, off, s[8:11], 0 glc
+; GFX1032-NEXT: s_load_dword s4, s[2:3], 0x0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s6
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: s_mul_i32 s11, s8, s5
+; GFX1032-NEXT: s_mov_b32 s6, -1
+; GFX1032-NEXT: s_mov_b32 s5, s3
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v0, s4
+; GFX1032-NEXT: s_mov_b32 s4, s2
+; GFX1032-NEXT: .LBB7_2: ; %atomicrmw.start
+; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: v_mov_b32_e32 v4, v0
+; GFX1032-NEXT: v_subrev_nc_u32_e32 v3, s11, v4
+; GFX1032-NEXT: v_mov_b32_e32 v0, v3
+; GFX1032-NEXT: v_mov_b32_e32 v1, v4
+; GFX1032-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: buffer_gl1_inv
; GFX1032-NEXT: buffer_gl0_inv
-; GFX1032-NEXT: .LBB7_2:
-; GFX1032-NEXT: s_waitcnt_depctr 0xffe3
-; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX1032-NEXT: s_or_b32 s10, vcc_lo, s10
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s10
+; GFX1032-NEXT: s_cbranch_execnz .LBB7_2
+; GFX1032-NEXT: ; %bb.3: ; %Flow
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s10
+; GFX1032-NEXT: .LBB7_4: ; %Flow3
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s9
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mul_lo_u32 v0, s6, v0
-; GFX1032-NEXT: v_readfirstlane_b32 s2, v1
+; GFX1032-NEXT: v_mul_lo_u32 v1, s8, v2
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
; GFX1032-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1032-NEXT: v_sub_nc_u32_e32 v0, s2, v0
+; GFX1032-NEXT: v_sub_nc_u32_e32 v0, s2, v1
; GFX1032-NEXT: s_mov_b32 s2, -1
; GFX1032-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX1032-NEXT: s_endpgm
@@ -4046,35 +4231,54 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1164: ; %bb.0: ; %entry
; GFX1164-NEXT: s_clause 0x1
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-NEXT: s_load_b32 s8, s[4:5], 0x34
+; GFX1164-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1164-NEXT: s_mov_b64 s[6:7], exec
-; GFX1164-NEXT: s_mov_b64 s[4:5], exec
+; GFX1164-NEXT: s_mov_b64 s[8:9], exec
; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164-NEXT: s_cbranch_execz .LBB7_2
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1164-NEXT: ; implicit-def: $vgpr0
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1164-NEXT: s_cbranch_execz .LBB7_4
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1164-NEXT: s_mov_b32 s15, 0x31016000
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_mul_i32 s6, s8, s6
-; GFX1164-NEXT: s_mov_b32 s14, -1
-; GFX1164-NEXT: v_mov_b32_e32 v1, s6
-; GFX1164-NEXT: s_mov_b32 s12, s2
-; GFX1164-NEXT: s_mov_b32 s13, s3
-; GFX1164-NEXT: buffer_atomic_sub_u32 v1, off, s[12:15], 0 glc
+; GFX1164-NEXT: s_load_b32 s4, s[2:3], 0x0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s5, s[6:7]
+; GFX1164-NEXT: s_mov_b64 s[10:11], 0
+; GFX1164-NEXT: s_mul_i32 s13, s12, s5
+; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-NEXT: s_mov_b32 s6, -1
+; GFX1164-NEXT: s_mov_b32 s5, s3
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: v_mov_b32_e32 v0, s4
+; GFX1164-NEXT: s_mov_b32 s4, s2
+; GFX1164-NEXT: .LBB7_2: ; %atomicrmw.start
+; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_mov_b32_e32 v4, v0
+; GFX1164-NEXT: v_subrev_nc_u32_e32 v3, s13, v4
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164-NEXT: v_mov_b32_e32 v0, v3
+; GFX1164-NEXT: v_mov_b32_e32 v1, v4
+; GFX1164-NEXT: buffer_atomic_cmpswap_b32 v[0:1], off, s[4:7], 0 glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: buffer_gl1_inv
; GFX1164-NEXT: buffer_gl0_inv
-; GFX1164-NEXT: .LBB7_2:
-; GFX1164-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX1164-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX1164-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1164-NEXT: s_cbranch_execnz .LBB7_2
+; GFX1164-NEXT: ; %bb.3: ; %Flow
+; GFX1164-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1164-NEXT: .LBB7_4: ; %Flow3
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mul_lo_u32 v0, s8, v0
-; GFX1164-NEXT: v_readfirstlane_b32 s2, v1
+; GFX1164-NEXT: v_mul_lo_u32 v1, s12, v2
+; GFX1164-NEXT: v_readfirstlane_b32 s2, v0
; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1164-NEXT: v_sub_nc_u32_e32 v0, s2, v0
+; GFX1164-NEXT: v_sub_nc_u32_e32 v0, s2, v1
; GFX1164-NEXT: s_mov_b32 s2, -1
; GFX1164-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX1164-NEXT: s_endpgm
@@ -4083,34 +4287,52 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1132: ; %bb.0: ; %entry
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132-NEXT: s_load_b32 s4, s[4:5], 0x34
+; GFX1132-NEXT: s_load_b32 s8, s[4:5], 0x34
; GFX1132-NEXT: s_mov_b32 s6, exec_lo
-; GFX1132-NEXT: s_mov_b32 s5, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1132-NEXT: ; implicit-def: $vgpr1
+; GFX1132-NEXT: s_mov_b32 s10, 0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
+; GFX1132-NEXT: s_mov_b32 s9, exec_lo
+; GFX1132-NEXT: ; implicit-def: $vgpr0
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132-NEXT: s_cbranch_execz .LBB7_2
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
+; GFX1132-NEXT: s_cbranch_execz .LBB7_4
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s6, s6
-; GFX1132-NEXT: s_mov_b32 s11, 0x31016000
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_mul_i32 s6, s4, s6
-; GFX1132-NEXT: s_mov_b32 s10, -1
-; GFX1132-NEXT: v_mov_b32_e32 v1, s6
-; GFX1132-NEXT: s_mov_b32 s8, s2
-; GFX1132-NEXT: s_mov_b32 s9, s3
-; GFX1132-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc
+; GFX1132-NEXT: s_load_b32 s4, s[2:3], 0x0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s6
+; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-NEXT: s_mul_i32 s11, s8, s5
+; GFX1132-NEXT: s_mov_b32 s6, -1
+; GFX1132-NEXT: s_mov_b32 s5, s3
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: v_mov_b32_e32 v0, s4
+; GFX1132-NEXT: s_mov_b32 s4, s2
+; GFX1132-NEXT: .LBB7_2: ; %atomicrmw.start
+; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_mov_b32_e32 v4, v0
+; GFX1132-NEXT: v_subrev_nc_u32_e32 v3, s11, v4
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-NEXT: v_dual_mov_b32 v0, v3 :: v_dual_mov_b32 v1, v4
+; GFX1132-NEXT: buffer_atomic_cmpswap_b32 v[0:1], off, s[4:7], 0 glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: buffer_gl1_inv
; GFX1132-NEXT: buffer_gl0_inv
-; GFX1132-NEXT: .LBB7_2:
-; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX1132-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX1132-NEXT: s_or_b32 s10, vcc_lo, s10
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s10
+; GFX1132-NEXT: s_cbranch_execnz .LBB7_2
+; GFX1132-NEXT: ; %bb.3: ; %Flow
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s10
+; GFX1132-NEXT: .LBB7_4: ; %Flow3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s9
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_mul_lo_u32 v0, s4, v0
-; GFX1132-NEXT: v_readfirstlane_b32 s2, v1
+; GFX1132-NEXT: v_mul_lo_u32 v1, s8, v2
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v0
; GFX1132-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1132-NEXT: v_sub_nc_u32_e32 v0, s2, v0
+; GFX1132-NEXT: v_sub_nc_u32_e32 v0, s2, v1
; GFX1132-NEXT: s_mov_b32 s2, -1
; GFX1132-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX1132-NEXT: s_endpgm
@@ -4190,7 +4412,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1232-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX1232-NEXT: s_endpgm
entry:
- %old = atomicrmw sub ptr addrspace(1) %inout, i32 %subitive syncscope("agent") acq_rel, !amdgpu.no.fine.grained.memory !0
+ %old = atomicrmw sub ptr addrspace(1) %inout, i32 %subitive syncscope("agent") acq_rel
store i32 %old, ptr addrspace(1) %out
ret void
}
@@ -4199,19 +4421,19 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS_ITERATIVE-LABEL: sub_i32_varying:
; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s6, 0
-; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr1
+; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s12, 0
+; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr2
; GFX7LESS_ITERATIVE-NEXT: .LBB8_1: ; %ComputeLoop
; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 m0, s2
-; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s7, v0, s2
-; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v1, s6, m0
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s2
+; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v2, s12, m0
; GFX7LESS_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
; GFX7LESS_ITERATIVE-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GFX7LESS_ITERATIVE-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX7LESS_ITERATIVE-NEXT: s_add_i32 s6, s6, s7
+; GFX7LESS_ITERATIVE-NEXT: s_add_i32 s12, s12, s6
; GFX7LESS_ITERATIVE-NEXT: s_cbranch_vccnz .LBB8_1
; GFX7LESS_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX7LESS_ITERATIVE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
@@ -4220,42 +4442,58 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr0
; GFX7LESS_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX7LESS_ITERATIVE-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GFX7LESS_ITERATIVE-NEXT: s_cbranch_execz .LBB8_4
+; GFX7LESS_ITERATIVE-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GFX7LESS_ITERATIVE-NEXT: s_cbranch_execz .LBB8_6
; GFX7LESS_ITERATIVE-NEXT: ; %bb.3:
-; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s11, 0xf000
-; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s10, -1
; GFX7LESS_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s8, s2
-; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s9, s3
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, s6
-; GFX7LESS_ITERATIVE-NEXT: buffer_atomic_sub v0, off, s[8:11], 0 glc
+; GFX7LESS_ITERATIVE-NEXT: s_load_dword s4, s[2:3], 0x0
+; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[10:11], 0
+; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, s4
+; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s6, -1
+; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s4, s2
+; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s5, s3
+; GFX7LESS_ITERATIVE-NEXT: .LBB8_4: ; %atomicrmw.start
+; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v4, v0
+; GFX7LESS_ITERATIVE-NEXT: v_subrev_i32_e32 v3, vcc, s12, v4
+; GFX7LESS_ITERATIVE-NEXT: s_waitcnt expcnt(0)
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, v3
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, v4
+; GFX7LESS_ITERATIVE-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
; GFX7LESS_ITERATIVE-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS_ITERATIVE-NEXT: buffer_wbinvl1
-; GFX7LESS_ITERATIVE-NEXT: .LBB8_4:
-; GFX7LESS_ITERATIVE-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX7LESS_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX7LESS_ITERATIVE-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX7LESS_ITERATIVE-NEXT: s_cbranch_execnz .LBB8_4
+; GFX7LESS_ITERATIVE-NEXT: ; %bb.5: ; %Flow
+; GFX7LESS_ITERATIVE-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX7LESS_ITERATIVE-NEXT: .LBB8_6: ; %Flow4
+; GFX7LESS_ITERATIVE-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7LESS_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s2, -1
; GFX7LESS_ITERATIVE-NEXT: v_readfirstlane_b32 s4, v0
; GFX7LESS_ITERATIVE-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS_ITERATIVE-NEXT: v_sub_i32_e32 v0, vcc, s4, v1
+; GFX7LESS_ITERATIVE-NEXT: v_sub_i32_e32 v0, vcc, s4, v2
; GFX7LESS_ITERATIVE-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7LESS_ITERATIVE-NEXT: s_endpgm
;
; GFX8_ITERATIVE-LABEL: sub_i32_varying:
; GFX8_ITERATIVE: ; %bb.0: ; %entry
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
-; GFX8_ITERATIVE-NEXT: s_mov_b32 s6, 0
-; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr1
+; GFX8_ITERATIVE-NEXT: s_mov_b32 s12, 0
+; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr2
; GFX8_ITERATIVE-NEXT: .LBB8_1: ; %ComputeLoop
; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s2
-; GFX8_ITERATIVE-NEXT: v_readlane_b32 s7, v0, s2
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s2
; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX8_ITERATIVE-NEXT: v_writelane_b32 v1, s6, m0
-; GFX8_ITERATIVE-NEXT: s_add_i32 s6, s6, s7
+; GFX8_ITERATIVE-NEXT: v_writelane_b32 v2, s12, m0
+; GFX8_ITERATIVE-NEXT: s_add_i32 s12, s12, s6
; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
; GFX8_ITERATIVE-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB8_1
@@ -4266,41 +4504,56 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX8_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr0
; GFX8_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX8_ITERATIVE-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GFX8_ITERATIVE-NEXT: s_cbranch_execz .LBB8_4
+; GFX8_ITERATIVE-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GFX8_ITERATIVE-NEXT: s_cbranch_execz .LBB8_6
; GFX8_ITERATIVE-NEXT: ; %bb.3:
-; GFX8_ITERATIVE-NEXT: s_mov_b32 s11, 0xf000
-; GFX8_ITERATIVE-NEXT: s_mov_b32 s10, -1
; GFX8_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8_ITERATIVE-NEXT: s_mov_b32 s8, s2
-; GFX8_ITERATIVE-NEXT: s_mov_b32 s9, s3
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, s6
-; GFX8_ITERATIVE-NEXT: buffer_atomic_sub v0, off, s[8:11], 0 glc
+; GFX8_ITERATIVE-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX8_ITERATIVE-NEXT: s_mov_b64 s[10:11], 0
+; GFX8_ITERATIVE-NEXT: s_mov_b32 s7, 0xf000
+; GFX8_ITERATIVE-NEXT: s_mov_b32 s6, -1
+; GFX8_ITERATIVE-NEXT: s_mov_b32 s4, s2
+; GFX8_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, s5
+; GFX8_ITERATIVE-NEXT: s_mov_b32 s5, s3
+; GFX8_ITERATIVE-NEXT: .LBB8_4: ; %atomicrmw.start
+; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, v0
+; GFX8_ITERATIVE-NEXT: v_subrev_u32_e32 v3, vcc, s12, v4
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, v3
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, v4
+; GFX8_ITERATIVE-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
; GFX8_ITERATIVE-NEXT: s_waitcnt vmcnt(0)
; GFX8_ITERATIVE-NEXT: buffer_wbinvl1_vol
-; GFX8_ITERATIVE-NEXT: .LBB8_4:
-; GFX8_ITERATIVE-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8_ITERATIVE-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX8_ITERATIVE-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX8_ITERATIVE-NEXT: s_cbranch_execnz .LBB8_4
+; GFX8_ITERATIVE-NEXT: ; %bb.5: ; %Flow
+; GFX8_ITERATIVE-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX8_ITERATIVE-NEXT: .LBB8_6: ; %Flow4
+; GFX8_ITERATIVE-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8_ITERATIVE-NEXT: v_readfirstlane_b32 s4, v0
; GFX8_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX8_ITERATIVE-NEXT: s_mov_b32 s3, 0xf000
; GFX8_ITERATIVE-NEXT: s_mov_b32 s2, -1
-; GFX8_ITERATIVE-NEXT: v_sub_u32_e32 v0, vcc, s4, v1
+; GFX8_ITERATIVE-NEXT: v_sub_u32_e32 v0, vcc, s4, v2
; GFX8_ITERATIVE-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX8_ITERATIVE-NEXT: s_endpgm
;
; GFX9_ITERATIVE-LABEL: sub_i32_varying:
; GFX9_ITERATIVE: ; %bb.0: ; %entry
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
-; GFX9_ITERATIVE-NEXT: s_mov_b32 s6, 0
-; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr1
+; GFX9_ITERATIVE-NEXT: s_mov_b32 s12, 0
+; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr2
; GFX9_ITERATIVE-NEXT: .LBB8_1: ; %ComputeLoop
; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s2
-; GFX9_ITERATIVE-NEXT: v_readlane_b32 s7, v0, s2
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s2
; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s2
-; GFX9_ITERATIVE-NEXT: v_writelane_b32 v1, s6, m0
-; GFX9_ITERATIVE-NEXT: s_add_i32 s6, s6, s7
+; GFX9_ITERATIVE-NEXT: v_writelane_b32 v2, s12, m0
+; GFX9_ITERATIVE-NEXT: s_add_i32 s12, s12, s6
; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
; GFX9_ITERATIVE-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB8_1
@@ -4311,41 +4564,56 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX9_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr0
; GFX9_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX9_ITERATIVE-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GFX9_ITERATIVE-NEXT: s_cbranch_execz .LBB8_4
+; GFX9_ITERATIVE-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GFX9_ITERATIVE-NEXT: s_cbranch_execz .LBB8_6
; GFX9_ITERATIVE-NEXT: ; %bb.3:
-; GFX9_ITERATIVE-NEXT: s_mov_b32 s11, 0xf000
-; GFX9_ITERATIVE-NEXT: s_mov_b32 s10, -1
; GFX9_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9_ITERATIVE-NEXT: s_mov_b32 s8, s2
-; GFX9_ITERATIVE-NEXT: s_mov_b32 s9, s3
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, s6
-; GFX9_ITERATIVE-NEXT: buffer_atomic_sub v0, off, s[8:11], 0 glc
+; GFX9_ITERATIVE-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX9_ITERATIVE-NEXT: s_mov_b64 s[10:11], 0
+; GFX9_ITERATIVE-NEXT: s_mov_b32 s7, 0xf000
+; GFX9_ITERATIVE-NEXT: s_mov_b32 s6, -1
+; GFX9_ITERATIVE-NEXT: s_mov_b32 s4, s2
+; GFX9_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, s5
+; GFX9_ITERATIVE-NEXT: s_mov_b32 s5, s3
+; GFX9_ITERATIVE-NEXT: .LBB8_4: ; %atomicrmw.start
+; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, v0
+; GFX9_ITERATIVE-NEXT: v_subrev_u32_e32 v3, s12, v4
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, v3
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, v4
+; GFX9_ITERATIVE-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
; GFX9_ITERATIVE-NEXT: s_waitcnt vmcnt(0)
; GFX9_ITERATIVE-NEXT: buffer_wbinvl1_vol
-; GFX9_ITERATIVE-NEXT: .LBB8_4:
-; GFX9_ITERATIVE-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX9_ITERATIVE-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX9_ITERATIVE-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX9_ITERATIVE-NEXT: s_cbranch_execnz .LBB8_4
+; GFX9_ITERATIVE-NEXT: ; %bb.5: ; %Flow
+; GFX9_ITERATIVE-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX9_ITERATIVE-NEXT: .LBB8_6: ; %Flow4
+; GFX9_ITERATIVE-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX9_ITERATIVE-NEXT: v_readfirstlane_b32 s4, v0
; GFX9_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX9_ITERATIVE-NEXT: s_mov_b32 s3, 0xf000
; GFX9_ITERATIVE-NEXT: s_mov_b32 s2, -1
-; GFX9_ITERATIVE-NEXT: v_sub_u32_e32 v0, s4, v1
+; GFX9_ITERATIVE-NEXT: v_sub_u32_e32 v0, s4, v2
; GFX9_ITERATIVE-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX9_ITERATIVE-NEXT: s_endpgm
;
; GFX1064_ITERATIVE-LABEL: sub_i32_varying:
; GFX1064_ITERATIVE: ; %bb.0: ; %entry
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064_ITERATIVE-NEXT: s_mov_b32 s6, 0
-; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr1
+; GFX1064_ITERATIVE-NEXT: s_mov_b32 s12, 0
+; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr2
; GFX1064_ITERATIVE-NEXT: .LBB8_1: ; %ComputeLoop
; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s7, s[0:1]
-; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s7
-; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v1, s6, s7
+; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s6, s[0:1]
+; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s7, v0, s6
+; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s6
+; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v2, s12, s6
; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX1064_ITERATIVE-NEXT: s_add_i32 s6, s6, s8
+; GFX1064_ITERATIVE-NEXT: s_add_i32 s12, s12, s7
; GFX1064_ITERATIVE-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX1064_ITERATIVE-NEXT: s_cbranch_scc1 .LBB8_1
; GFX1064_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
@@ -4355,26 +4623,40 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1064_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr0
; GFX1064_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX1064_ITERATIVE-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GFX1064_ITERATIVE-NEXT: s_cbranch_execz .LBB8_4
+; GFX1064_ITERATIVE-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GFX1064_ITERATIVE-NEXT: s_cbranch_execz .LBB8_6
; GFX1064_ITERATIVE-NEXT: ; %bb.3:
-; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v0, s6
-; GFX1064_ITERATIVE-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1064_ITERATIVE-NEXT: s_mov_b32 s10, -1
; GFX1064_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064_ITERATIVE-NEXT: s_mov_b32 s8, s2
-; GFX1064_ITERATIVE-NEXT: s_mov_b32 s9, s3
-; GFX1064_ITERATIVE-NEXT: buffer_atomic_sub v0, off, s[8:11], 0 glc
+; GFX1064_ITERATIVE-NEXT: s_load_dword s4, s[2:3], 0x0
+; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[10:11], 0
+; GFX1064_ITERATIVE-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064_ITERATIVE-NEXT: s_mov_b32 s6, -1
+; GFX1064_ITERATIVE-NEXT: s_mov_b32 s5, s3
+; GFX1064_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v0, s4
+; GFX1064_ITERATIVE-NEXT: s_mov_b32 s4, s2
+; GFX1064_ITERATIVE-NEXT: .LBB8_4: ; %atomicrmw.start
+; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v4, v0
+; GFX1064_ITERATIVE-NEXT: v_subrev_nc_u32_e32 v3, s12, v4
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v0, v3
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v1, v4
+; GFX1064_ITERATIVE-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
; GFX1064_ITERATIVE-NEXT: s_waitcnt vmcnt(0)
; GFX1064_ITERATIVE-NEXT: buffer_gl1_inv
; GFX1064_ITERATIVE-NEXT: buffer_gl0_inv
-; GFX1064_ITERATIVE-NEXT: .LBB8_4:
-; GFX1064_ITERATIVE-NEXT: s_waitcnt_depctr 0xffe3
-; GFX1064_ITERATIVE-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX1064_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX1064_ITERATIVE-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX1064_ITERATIVE-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX1064_ITERATIVE-NEXT: s_cbranch_execnz .LBB8_4
+; GFX1064_ITERATIVE-NEXT: ; %bb.5: ; %Flow
+; GFX1064_ITERATIVE-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1064_ITERATIVE-NEXT: .LBB8_6: ; %Flow4
+; GFX1064_ITERATIVE-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1064_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v0
; GFX1064_ITERATIVE-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1064_ITERATIVE-NEXT: v_sub_nc_u32_e32 v0, s2, v1
+; GFX1064_ITERATIVE-NEXT: v_sub_nc_u32_e32 v0, s2, v2
; GFX1064_ITERATIVE-NEXT: s_mov_b32 s2, -1
; GFX1064_ITERATIVE-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX1064_ITERATIVE-NEXT: s_endpgm
@@ -4382,140 +4664,191 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1032_ITERATIVE-LABEL: sub_i32_varying:
; GFX1032_ITERATIVE: ; %bb.0: ; %entry
; GFX1032_ITERATIVE-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s6, 0
-; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr1
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s8, 0
+; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr2
; GFX1032_ITERATIVE-NEXT: .LBB8_1: ; %ComputeLoop
; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s1, s0
; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s2, v0, s1
; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s1
-; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s6, s1
+; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s8, s1
; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s0, s0, s3
-; GFX1032_ITERATIVE-NEXT: s_add_i32 s6, s6, s2
+; GFX1032_ITERATIVE-NEXT: s_add_i32 s8, s8, s2
; GFX1032_ITERATIVE-NEXT: s_cmp_lg_u32 s0, 0
; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB8_1
; GFX1032_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1032_ITERATIVE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s10, 0
; GFX1032_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr0
; GFX1032_ITERATIVE-NEXT: s_and_saveexec_b32 s4, vcc_lo
-; GFX1032_ITERATIVE-NEXT: s_xor_b32 s4, exec_lo, s4
-; GFX1032_ITERATIVE-NEXT: s_cbranch_execz .LBB8_4
+; GFX1032_ITERATIVE-NEXT: s_xor_b32 s9, exec_lo, s4
+; GFX1032_ITERATIVE-NEXT: s_cbranch_execz .LBB8_6
; GFX1032_ITERATIVE-NEXT: ; %bb.3:
-; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v0, s6
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s10, -1
; GFX1032_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s8, s2
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s9, s3
-; GFX1032_ITERATIVE-NEXT: buffer_atomic_sub v0, off, s[8:11], 0 glc
+; GFX1032_ITERATIVE-NEXT: s_load_dword s4, s[2:3], 0x0
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s6, -1
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s5, s3
+; GFX1032_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v0, s4
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s4, s2
+; GFX1032_ITERATIVE-NEXT: .LBB8_4: ; %atomicrmw.start
+; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v4, v0
+; GFX1032_ITERATIVE-NEXT: v_subrev_nc_u32_e32 v3, s8, v4
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v0, v3
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v1, v4
+; GFX1032_ITERATIVE-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
; GFX1032_ITERATIVE-NEXT: s_waitcnt vmcnt(0)
; GFX1032_ITERATIVE-NEXT: buffer_gl1_inv
; GFX1032_ITERATIVE-NEXT: buffer_gl0_inv
-; GFX1032_ITERATIVE-NEXT: .LBB8_4:
-; GFX1032_ITERATIVE-NEXT: s_waitcnt_depctr 0xffe3
-; GFX1032_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX1032_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX1032_ITERATIVE-NEXT: s_or_b32 s10, vcc_lo, s10
+; GFX1032_ITERATIVE-NEXT: s_andn2_b32 exec_lo, exec_lo, s10
+; GFX1032_ITERATIVE-NEXT: s_cbranch_execnz .LBB8_4
+; GFX1032_ITERATIVE-NEXT: ; %bb.5: ; %Flow
+; GFX1032_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s10
+; GFX1032_ITERATIVE-NEXT: .LBB8_6: ; %Flow4
+; GFX1032_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s9
; GFX1032_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v0
; GFX1032_ITERATIVE-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1032_ITERATIVE-NEXT: v_sub_nc_u32_e32 v0, s2, v1
+; GFX1032_ITERATIVE-NEXT: v_sub_nc_u32_e32 v0, s2, v2
; GFX1032_ITERATIVE-NEXT: s_mov_b32 s2, -1
; GFX1032_ITERATIVE-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX1032_ITERATIVE-NEXT: s_endpgm
;
; GFX1164_ITERATIVE-LABEL: sub_i32_varying:
; GFX1164_ITERATIVE: ; %bb.0: ; %entry
-; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164_ITERATIVE-NEXT: s_mov_b32 s6, 0
-; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr0
+; GFX1164_ITERATIVE-NEXT: s_mov_b32 s12, 0
+; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr2
; GFX1164_ITERATIVE-NEXT: .LBB8_1: ; %ComputeLoop
; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s7, s[0:1]
+; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s6, s[0:1]
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s8, v1, s7
-; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s7
-; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v0, s6, s7
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s7, v0, s6
+; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s6
+; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v2, s12, s6
; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
-; GFX1164_ITERATIVE-NEXT: s_add_i32 s6, s6, s8
+; GFX1164_ITERATIVE-NEXT: s_add_i32 s12, s12, s7
; GFX1164_ITERATIVE-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX1164_ITERATIVE-NEXT: s_cbranch_scc1 .LBB8_1
; GFX1164_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1164_ITERATIVE-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1164_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v1, exec_hi, v1
-; GFX1164_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr1
+; GFX1164_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr0
; GFX1164_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164_ITERATIVE-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GFX1164_ITERATIVE-NEXT: s_cbranch_execz .LBB8_4
+; GFX1164_ITERATIVE-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; GFX1164_ITERATIVE-NEXT: s_cbranch_execz .LBB8_6
; GFX1164_ITERATIVE-NEXT: ; %bb.3:
-; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v1, s6
-; GFX1164_ITERATIVE-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1164_ITERATIVE-NEXT: s_mov_b32 s10, -1
; GFX1164_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164_ITERATIVE-NEXT: s_mov_b32 s8, s2
-; GFX1164_ITERATIVE-NEXT: s_mov_b32 s9, s3
-; GFX1164_ITERATIVE-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc
+; GFX1164_ITERATIVE-NEXT: s_load_b32 s4, s[2:3], 0x0
+; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[10:11], 0
+; GFX1164_ITERATIVE-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164_ITERATIVE-NEXT: s_mov_b32 s6, -1
+; GFX1164_ITERATIVE-NEXT: s_mov_b32 s5, s3
+; GFX1164_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v0, s4
+; GFX1164_ITERATIVE-NEXT: s_mov_b32 s4, s2
+; GFX1164_ITERATIVE-NEXT: .LBB8_4: ; %atomicrmw.start
+; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v4, v0
+; GFX1164_ITERATIVE-NEXT: v_subrev_nc_u32_e32 v3, s12, v4
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v0, v3
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v1, v4
+; GFX1164_ITERATIVE-NEXT: buffer_atomic_cmpswap_b32 v[0:1], off, s[4:7], 0 glc
; GFX1164_ITERATIVE-NEXT: s_waitcnt vmcnt(0)
; GFX1164_ITERATIVE-NEXT: buffer_gl1_inv
; GFX1164_ITERATIVE-NEXT: buffer_gl0_inv
-; GFX1164_ITERATIVE-NEXT: .LBB8_4:
-; GFX1164_ITERATIVE-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX1164_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX1164_ITERATIVE-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1164_ITERATIVE-NEXT: s_cbranch_execnz .LBB8_4
+; GFX1164_ITERATIVE-NEXT: ; %bb.5: ; %Flow
+; GFX1164_ITERATIVE-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1164_ITERATIVE-NEXT: .LBB8_6: ; %Flow4
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164_ITERATIVE-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1164_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v1
+; GFX1164_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v0
; GFX1164_ITERATIVE-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1164_ITERATIVE-NEXT: v_sub_nc_u32_e32 v0, s2, v0
+; GFX1164_ITERATIVE-NEXT: v_sub_nc_u32_e32 v0, s2, v2
; GFX1164_ITERATIVE-NEXT: s_mov_b32 s2, -1
; GFX1164_ITERATIVE-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX1164_ITERATIVE-NEXT: s_endpgm
;
; GFX1132_ITERATIVE-LABEL: sub_i32_varying:
; GFX1132_ITERATIVE: ; %bb.0: ; %entry
-; GFX1132_ITERATIVE-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX1132_ITERATIVE-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1132_ITERATIVE-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s6, 0
-; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr0
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s8, 0
+; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr2
; GFX1132_ITERATIVE-NEXT: .LBB8_1: ; %ComputeLoop
; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s1, s0
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s2, v1, s1
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s2, v0, s1
; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s1
-; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s6, s1
+; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v2, s8, s1
; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s0, s0, s3
-; GFX1132_ITERATIVE-NEXT: s_add_i32 s6, s6, s2
+; GFX1132_ITERATIVE-NEXT: s_add_i32 s8, s8, s2
; GFX1132_ITERATIVE-NEXT: s_cmp_lg_u32 s0, 0
; GFX1132_ITERATIVE-NEXT: s_cbranch_scc1 .LBB8_1
; GFX1132_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1132_ITERATIVE-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
+; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s10, 0
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr1
+; GFX1132_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr0
; GFX1132_ITERATIVE-NEXT: s_and_saveexec_b32 s4, vcc_lo
-; GFX1132_ITERATIVE-NEXT: s_xor_b32 s4, exec_lo, s4
-; GFX1132_ITERATIVE-NEXT: s_cbranch_execz .LBB8_4
+; GFX1132_ITERATIVE-NEXT: s_xor_b32 s9, exec_lo, s4
+; GFX1132_ITERATIVE-NEXT: s_cbranch_execz .LBB8_6
; GFX1132_ITERATIVE-NEXT: ; %bb.3:
-; GFX1132_ITERATIVE-NEXT: v_mov_b32_e32 v1, s6
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s10, -1
; GFX1132_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s8, s2
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s9, s3
-; GFX1132_ITERATIVE-NEXT: buffer_atomic_sub_u32 v1, off, s[8:11], 0 glc
+; GFX1132_ITERATIVE-NEXT: s_load_b32 s4, s[2:3], 0x0
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s6, -1
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s5, s3
+; GFX1132_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132_ITERATIVE-NEXT: v_mov_b32_e32 v0, s4
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s4, s2
+; GFX1132_ITERATIVE-NEXT: .LBB8_4: ; %atomicrmw.start
+; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132_ITERATIVE-NEXT: v_mov_b32_e32 v4, v0
+; GFX1132_ITERATIVE-NEXT: v_subrev_nc_u32_e32 v3, s8, v4
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v0, v3 :: v_dual_mov_b32 v1, v4
+; GFX1132_ITERATIVE-NEXT: buffer_atomic_cmpswap_b32 v[0:1], off, s[4:7], 0 glc
; GFX1132_ITERATIVE-NEXT: s_waitcnt vmcnt(0)
; GFX1132_ITERATIVE-NEXT: buffer_gl1_inv
; GFX1132_ITERATIVE-NEXT: buffer_gl0_inv
-; GFX1132_ITERATIVE-NEXT: .LBB8_4:
-; GFX1132_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX1132_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX1132_ITERATIVE-NEXT: s_or_b32 s10, vcc_lo, s10
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 exec_lo, exec_lo, s10
+; GFX1132_ITERATIVE-NEXT: s_cbranch_execnz .LBB8_4
+; GFX1132_ITERATIVE-NEXT: ; %bb.5: ; %Flow
+; GFX1132_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s10
+; GFX1132_ITERATIVE-NEXT: .LBB8_6: ; %Flow4
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s9
; GFX1132_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v1
+; GFX1132_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v0
; GFX1132_ITERATIVE-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1132_ITERATIVE-NEXT: v_sub_nc_u32_e32 v0, s2, v0
+; GFX1132_ITERATIVE-NEXT: v_sub_nc_u32_e32 v0, s2, v2
; GFX1132_ITERATIVE-NEXT: s_mov_b32 s2, -1
; GFX1132_ITERATIVE-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX1132_ITERATIVE-NEXT: s_endpgm
@@ -4618,19 +4951,34 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS_DPP-LABEL: sub_i32_varying:
; GFX7LESS_DPP: ; %bb.0: ; %entry
; GFX7LESS_DPP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7LESS_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS_DPP-NEXT: s_load_dword s4, s[2:3], 0x0
+; GFX7LESS_DPP-NEXT: s_mov_b64 s[8:9], 0
; GFX7LESS_DPP-NEXT: s_mov_b32 s7, 0xf000
-; GFX7LESS_DPP-NEXT: s_mov_b32 s6, -1
-; GFX7LESS_DPP-NEXT: s_mov_b32 s10, s6
-; GFX7LESS_DPP-NEXT: s_mov_b32 s11, s7
; GFX7LESS_DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS_DPP-NEXT: s_mov_b32 s8, s2
-; GFX7LESS_DPP-NEXT: s_mov_b32 s9, s3
-; GFX7LESS_DPP-NEXT: buffer_atomic_sub v0, off, s[8:11], 0 glc
+; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v2, s4
+; GFX7LESS_DPP-NEXT: s_mov_b32 s6, -1
+; GFX7LESS_DPP-NEXT: s_mov_b32 s4, s2
+; GFX7LESS_DPP-NEXT: s_mov_b32 s5, s3
+; GFX7LESS_DPP-NEXT: .LBB8_1: ; %atomicrmw.start
+; GFX7LESS_DPP-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS_DPP-NEXT: v_sub_i32_e32 v1, vcc, v2, v0
+; GFX7LESS_DPP-NEXT: s_waitcnt expcnt(0)
+; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v4, v2
+; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS_DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[4:7], 0 glc
; GFX7LESS_DPP-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS_DPP-NEXT: buffer_wbinvl1
-; GFX7LESS_DPP-NEXT: s_mov_b32 s4, s0
-; GFX7LESS_DPP-NEXT: s_mov_b32 s5, s1
-; GFX7LESS_DPP-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX7LESS_DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX7LESS_DPP-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v2, v3
+; GFX7LESS_DPP-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX7LESS_DPP-NEXT: s_cbranch_execnz .LBB8_1
+; GFX7LESS_DPP-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX7LESS_DPP-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX7LESS_DPP-NEXT: s_mov_b32 s3, 0xf000
+; GFX7LESS_DPP-NEXT: s_mov_b32 s2, -1
+; GFX7LESS_DPP-NEXT: buffer_store_dword v3, off, s[0:3], 0
; GFX7LESS_DPP-NEXT: s_endpgm
;
; GFX8_DPP-LABEL: sub_i32_varying:
@@ -4655,27 +5003,42 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX8_DPP-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
; GFX8_DPP-NEXT: s_nop 1
; GFX8_DPP-NEXT: v_add_u32_dpp v2, vcc, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX8_DPP-NEXT: v_readlane_b32 s6, v2, 63
+; GFX8_DPP-NEXT: v_readlane_b32 s12, v2, 63
; GFX8_DPP-NEXT: s_nop 0
; GFX8_DPP-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: s_mov_b64 exec, s[4:5]
; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX8_DPP-NEXT: ; implicit-def: $vgpr0
-; GFX8_DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX8_DPP-NEXT: s_cbranch_execz .LBB8_2
+; GFX8_DPP-NEXT: ; implicit-def: $vgpr3
+; GFX8_DPP-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX8_DPP-NEXT: s_cbranch_execz .LBB8_4
; GFX8_DPP-NEXT: ; %bb.1:
-; GFX8_DPP-NEXT: s_mov_b32 s11, 0xf000
-; GFX8_DPP-NEXT: s_mov_b32 s10, -1
; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8_DPP-NEXT: s_mov_b32 s8, s2
-; GFX8_DPP-NEXT: s_mov_b32 s9, s3
-; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s6
-; GFX8_DPP-NEXT: buffer_atomic_sub v0, off, s[8:11], 0 glc
+; GFX8_DPP-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX8_DPP-NEXT: s_mov_b64 s[10:11], 0
+; GFX8_DPP-NEXT: s_mov_b32 s7, 0xf000
+; GFX8_DPP-NEXT: s_mov_b32 s6, -1
+; GFX8_DPP-NEXT: s_mov_b32 s4, s2
+; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8_DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX8_DPP-NEXT: s_mov_b32 s5, s3
+; GFX8_DPP-NEXT: .LBB8_2: ; %atomicrmw.start
+; GFX8_DPP-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8_DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX8_DPP-NEXT: v_subrev_u32_e32 v4, vcc, s12, v5
+; GFX8_DPP-NEXT: v_mov_b32_e32 v3, v4
+; GFX8_DPP-NEXT: v_mov_b32_e32 v4, v5
+; GFX8_DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[4:7], 0 glc
; GFX8_DPP-NEXT: s_waitcnt vmcnt(0)
; GFX8_DPP-NEXT: buffer_wbinvl1_vol
-; GFX8_DPP-NEXT: .LBB8_2:
-; GFX8_DPP-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX8_DPP-NEXT: v_readfirstlane_b32 s4, v0
+; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX8_DPP-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX8_DPP-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX8_DPP-NEXT: s_cbranch_execnz .LBB8_2
+; GFX8_DPP-NEXT: ; %bb.3: ; %Flow
+; GFX8_DPP-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX8_DPP-NEXT: .LBB8_4: ; %Flow3
+; GFX8_DPP-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX8_DPP-NEXT: v_readfirstlane_b32 s4, v3
; GFX8_DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX8_DPP-NEXT: s_mov_b32 s3, 0xf000
@@ -4706,27 +5069,42 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX9_DPP-NEXT: v_add_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xa bank_mask:0xf
; GFX9_DPP-NEXT: s_nop 1
; GFX9_DPP-NEXT: v_add_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
-; GFX9_DPP-NEXT: v_readlane_b32 s6, v2, 63
+; GFX9_DPP-NEXT: v_readlane_b32 s12, v2, 63
; GFX9_DPP-NEXT: s_nop 0
; GFX9_DPP-NEXT: v_mov_b32_dpp v1, v2 wave_shr:1 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: s_mov_b64 exec, s[4:5]
; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9_DPP-NEXT: ; implicit-def: $vgpr0
-; GFX9_DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX9_DPP-NEXT: s_cbranch_execz .LBB8_2
+; GFX9_DPP-NEXT: ; implicit-def: $vgpr3
+; GFX9_DPP-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX9_DPP-NEXT: s_cbranch_execz .LBB8_4
; GFX9_DPP-NEXT: ; %bb.1:
-; GFX9_DPP-NEXT: s_mov_b32 s11, 0xf000
-; GFX9_DPP-NEXT: s_mov_b32 s10, -1
; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9_DPP-NEXT: s_mov_b32 s8, s2
-; GFX9_DPP-NEXT: s_mov_b32 s9, s3
-; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s6
-; GFX9_DPP-NEXT: buffer_atomic_sub v0, off, s[8:11], 0 glc
+; GFX9_DPP-NEXT: s_load_dword s5, s[2:3], 0x0
+; GFX9_DPP-NEXT: s_mov_b64 s[10:11], 0
+; GFX9_DPP-NEXT: s_mov_b32 s7, 0xf000
+; GFX9_DPP-NEXT: s_mov_b32 s6, -1
+; GFX9_DPP-NEXT: s_mov_b32 s4, s2
+; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9_DPP-NEXT: v_mov_b32_e32 v3, s5
+; GFX9_DPP-NEXT: s_mov_b32 s5, s3
+; GFX9_DPP-NEXT: .LBB8_2: ; %atomicrmw.start
+; GFX9_DPP-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9_DPP-NEXT: v_mov_b32_e32 v5, v3
+; GFX9_DPP-NEXT: v_subrev_u32_e32 v4, s12, v5
+; GFX9_DPP-NEXT: v_mov_b32_e32 v3, v4
+; GFX9_DPP-NEXT: v_mov_b32_e32 v4, v5
+; GFX9_DPP-NEXT: buffer_atomic_cmpswap v[3:4], off, s[4:7], 0 glc
; GFX9_DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9_DPP-NEXT: buffer_wbinvl1_vol
-; GFX9_DPP-NEXT: .LBB8_2:
-; GFX9_DPP-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9_DPP-NEXT: v_readfirstlane_b32 s4, v0
+; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX9_DPP-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX9_DPP-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX9_DPP-NEXT: s_cbranch_execnz .LBB8_2
+; GFX9_DPP-NEXT: ; %bb.3: ; %Flow
+; GFX9_DPP-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX9_DPP-NEXT: .LBB8_4: ; %Flow3
+; GFX9_DPP-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX9_DPP-NEXT: v_readfirstlane_b32 s4, v3
; GFX9_DPP-NEXT: v_mov_b32_e32 v0, v1
; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9_DPP-NEXT: s_mov_b32 s3, 0xf000
@@ -4759,39 +5137,52 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1064_DPP-NEXT: s_mov_b64 exec, s[4:5]
; GFX1064_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[4:5], -1
-; GFX1064_DPP-NEXT: v_readlane_b32 s8, v1, 47
-; GFX1064_DPP-NEXT: v_readlane_b32 s9, v1, 63
+; GFX1064_DPP-NEXT: v_readlane_b32 s6, v1, 47
+; GFX1064_DPP-NEXT: v_readlane_b32 s12, v1, 63
; GFX1064_DPP-NEXT: v_writelane_b32 v3, s7, 32
; GFX1064_DPP-NEXT: s_mov_b64 exec, s[4:5]
; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[6:7], -1
-; GFX1064_DPP-NEXT: s_mov_b32 s4, s9
-; GFX1064_DPP-NEXT: v_writelane_b32 v3, s8, 48
-; GFX1064_DPP-NEXT: s_mov_b64 exec, s[6:7]
+; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[4:5], -1
+; GFX1064_DPP-NEXT: v_writelane_b32 v3, s6, 48
+; GFX1064_DPP-NEXT: s_mov_b64 exec, s[4:5]
; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064_DPP-NEXT: s_mov_b32 s6, -1
-; GFX1064_DPP-NEXT: ; implicit-def: $vgpr0
+; GFX1064_DPP-NEXT: ; implicit-def: $vgpr4
; GFX1064_DPP-NEXT: s_and_saveexec_b64 s[8:9], vcc
-; GFX1064_DPP-NEXT: s_cbranch_execz .LBB8_2
+; GFX1064_DPP-NEXT: s_cbranch_execz .LBB8_4
; GFX1064_DPP-NEXT: ; %bb.1:
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v0, s4
+; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064_DPP-NEXT: s_load_dword s4, s[2:3], 0x0
+; GFX1064_DPP-NEXT: s_mov_b64 s[10:11], 0
; GFX1064_DPP-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064_DPP-NEXT: s_mov_b32 s5, s3
; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, s4
; GFX1064_DPP-NEXT: s_mov_b32 s4, s2
-; GFX1064_DPP-NEXT: s_mov_b32 s5, s3
-; GFX1064_DPP-NEXT: buffer_atomic_sub v0, off, s[4:7], 0 glc
+; GFX1064_DPP-NEXT: .LBB8_2: ; %atomicrmw.start
+; GFX1064_DPP-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1064_DPP-NEXT: v_subrev_nc_u32_e32 v5, s12, v6
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v4, v5
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v5, v6
+; GFX1064_DPP-NEXT: buffer_atomic_cmpswap v[4:5], off, s[4:7], 0 glc
; GFX1064_DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064_DPP-NEXT: buffer_gl1_inv
; GFX1064_DPP-NEXT: buffer_gl0_inv
-; GFX1064_DPP-NEXT: .LBB8_2:
-; GFX1064_DPP-NEXT: s_waitcnt_depctr 0xffe3
+; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
+; GFX1064_DPP-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX1064_DPP-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX1064_DPP-NEXT: s_cbranch_execnz .LBB8_2
+; GFX1064_DPP-NEXT: ; %bb.3: ; %Flow
+; GFX1064_DPP-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1064_DPP-NEXT: .LBB8_4: ; %Flow3
; GFX1064_DPP-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064_DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1064_DPP-NEXT: v_readfirstlane_b32 s2, v4
; GFX1064_DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1064_DPP-NEXT: s_mov_b32 s3, 0x31016000
; GFX1064_DPP-NEXT: v_sub_nc_u32_e32 v0, s2, v0
-; GFX1064_DPP-NEXT: s_mov_b32 s2, s6
+; GFX1064_DPP-NEXT: s_mov_b32 s2, -1
; GFX1064_DPP-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX1064_DPP-NEXT: s_endpgm
;
@@ -4809,7 +5200,7 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -1
; GFX1032_DPP-NEXT: v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
; GFX1032_DPP-NEXT: v_mov_b32_e32 v3, 0
-; GFX1032_DPP-NEXT: v_readlane_b32 s6, v1, 31
+; GFX1032_DPP-NEXT: v_readlane_b32 s9, v1, 31
; GFX1032_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1032_DPP-NEXT: v_readlane_b32 s5, v1, 15
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4
@@ -4818,30 +5209,43 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1032_DPP-NEXT: v_writelane_b32 v3, s5, 16
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4
; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032_DPP-NEXT: s_mov_b32 s4, s6
; GFX1032_DPP-NEXT: s_mov_b32 s6, -1
-; GFX1032_DPP-NEXT: ; implicit-def: $vgpr0
+; GFX1032_DPP-NEXT: ; implicit-def: $vgpr4
; GFX1032_DPP-NEXT: s_and_saveexec_b32 s8, vcc_lo
-; GFX1032_DPP-NEXT: s_cbranch_execz .LBB8_2
+; GFX1032_DPP-NEXT: s_cbranch_execz .LBB8_4
; GFX1032_DPP-NEXT: ; %bb.1:
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, s4
+; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032_DPP-NEXT: s_load_dword s4, s[2:3], 0x0
+; GFX1032_DPP-NEXT: s_mov_b32 s10, 0
; GFX1032_DPP-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032_DPP-NEXT: s_mov_b32 s5, s3
; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, s4
; GFX1032_DPP-NEXT: s_mov_b32 s4, s2
-; GFX1032_DPP-NEXT: s_mov_b32 s5, s3
-; GFX1032_DPP-NEXT: buffer_atomic_sub v0, off, s[4:7], 0 glc
+; GFX1032_DPP-NEXT: .LBB8_2: ; %atomicrmw.start
+; GFX1032_DPP-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1032_DPP-NEXT: v_subrev_nc_u32_e32 v5, s9, v6
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v4, v5
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v5, v6
+; GFX1032_DPP-NEXT: buffer_atomic_cmpswap v[4:5], off, s[4:7], 0 glc
; GFX1032_DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032_DPP-NEXT: buffer_gl1_inv
; GFX1032_DPP-NEXT: buffer_gl0_inv
-; GFX1032_DPP-NEXT: .LBB8_2:
-; GFX1032_DPP-NEXT: s_waitcnt_depctr 0xffe3
+; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX1032_DPP-NEXT: s_or_b32 s10, vcc_lo, s10
+; GFX1032_DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s10
+; GFX1032_DPP-NEXT: s_cbranch_execnz .LBB8_2
+; GFX1032_DPP-NEXT: ; %bb.3: ; %Flow
+; GFX1032_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s10
+; GFX1032_DPP-NEXT: .LBB8_4: ; %Flow3
; GFX1032_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032_DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032_DPP-NEXT: v_readfirstlane_b32 s2, v4
; GFX1032_DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1032_DPP-NEXT: s_mov_b32 s3, 0x31016000
; GFX1032_DPP-NEXT: v_sub_nc_u32_e32 v0, s2, v0
-; GFX1032_DPP-NEXT: s_mov_b32 s2, s6
+; GFX1032_DPP-NEXT: s_mov_b32 s2, -1
; GFX1032_DPP-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX1032_DPP-NEXT: s_endpgm
;
@@ -4877,40 +5281,57 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[4:5], -1
-; GFX1164_DPP-NEXT: v_readlane_b32 s8, v1, 47
-; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 63
+; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 47
+; GFX1164_DPP-NEXT: v_readlane_b32 s12, v1, 63
; GFX1164_DPP-NEXT: v_writelane_b32 v3, s7, 32
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[4:5]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[6:7], -1
-; GFX1164_DPP-NEXT: s_mov_b32 s4, s9
-; GFX1164_DPP-NEXT: v_writelane_b32 v3, s8, 48
-; GFX1164_DPP-NEXT: s_mov_b64 exec, s[6:7]
-; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[4:5], -1
+; GFX1164_DPP-NEXT: v_writelane_b32 v3, s6, 48
+; GFX1164_DPP-NEXT: s_mov_b64 exec, s[4:5]
; GFX1164_DPP-NEXT: s_mov_b32 s6, -1
-; GFX1164_DPP-NEXT: ; implicit-def: $vgpr0
-; GFX1164_DPP-NEXT: s_and_saveexec_b64 s[8:9], vcc
-; GFX1164_DPP-NEXT: s_cbranch_execz .LBB8_2
+; GFX1164_DPP-NEXT: s_mov_b64 s[8:9], exec
+; GFX1164_DPP-NEXT: ; implicit-def: $vgpr4
+; GFX1164_DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1164_DPP-NEXT: s_cbranch_execz .LBB8_4
; GFX1164_DPP-NEXT: ; %bb.1:
-; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, s4
+; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164_DPP-NEXT: s_load_b32 s4, s[2:3], 0x0
+; GFX1164_DPP-NEXT: s_mov_b64 s[10:11], 0
; GFX1164_DPP-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164_DPP-NEXT: s_mov_b32 s5, s3
; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, s4
; GFX1164_DPP-NEXT: s_mov_b32 s4, s2
-; GFX1164_DPP-NEXT: s_mov_b32 s5, s3
-; GFX1164_DPP-NEXT: buffer_atomic_sub_u32 v0, off, s[4:7], 0 glc
+; GFX1164_DPP-NEXT: .LBB8_2: ; %atomicrmw.start
+; GFX1164_DPP-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1164_DPP-NEXT: v_subrev_nc_u32_e32 v5, s12, v6
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v4, v5
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v5, v6
+; GFX1164_DPP-NEXT: buffer_atomic_cmpswap_b32 v[4:5], off, s[4:7], 0 glc
; GFX1164_DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164_DPP-NEXT: buffer_gl1_inv
; GFX1164_DPP-NEXT: buffer_gl0_inv
-; GFX1164_DPP-NEXT: .LBB8_2:
+; GFX1164_DPP-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
+; GFX1164_DPP-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164_DPP-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1164_DPP-NEXT: s_cbranch_execnz .LBB8_2
+; GFX1164_DPP-NEXT: ; %bb.3: ; %Flow
+; GFX1164_DPP-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1164_DPP-NEXT: .LBB8_4: ; %Flow3
+; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1164_DPP-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164_DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1164_DPP-NEXT: v_readfirstlane_b32 s2, v4
; GFX1164_DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1164_DPP-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164_DPP-NEXT: v_sub_nc_u32_e32 v0, s2, v0
-; GFX1164_DPP-NEXT: s_mov_b32 s2, s6
+; GFX1164_DPP-NEXT: s_mov_b32 s2, -1
; GFX1164_DPP-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX1164_DPP-NEXT: s_endpgm
;
@@ -4934,40 +5355,57 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1132_DPP-NEXT: v_add_nc_u32_dpp v1, v2, v1 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
; GFX1132_DPP-NEXT: v_mov_b32_e32 v3, 0
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1132_DPP-NEXT: v_readlane_b32 s6, v1, 31
+; GFX1132_DPP-NEXT: v_readlane_b32 s9, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1132_DPP-NEXT: v_readlane_b32 s5, v1, 15
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4
-; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -1
; GFX1132_DPP-NEXT: v_writelane_b32 v3, s5, 16
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4
-; GFX1132_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1132_DPP-NEXT: s_mov_b32 s4, s6
; GFX1132_DPP-NEXT: s_mov_b32 s6, -1
-; GFX1132_DPP-NEXT: ; implicit-def: $vgpr0
-; GFX1132_DPP-NEXT: s_and_saveexec_b32 s8, vcc_lo
-; GFX1132_DPP-NEXT: s_cbranch_execz .LBB8_2
+; GFX1132_DPP-NEXT: s_mov_b32 s8, exec_lo
+; GFX1132_DPP-NEXT: ; implicit-def: $vgpr4
+; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
+; GFX1132_DPP-NEXT: s_cbranch_execz .LBB8_4
; GFX1132_DPP-NEXT: ; %bb.1:
-; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, s4
+; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132_DPP-NEXT: s_load_b32 s4, s[2:3], 0x0
+; GFX1132_DPP-NEXT: s_mov_b32 s10, 0
; GFX1132_DPP-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132_DPP-NEXT: s_mov_b32 s5, s3
; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, s4
; GFX1132_DPP-NEXT: s_mov_b32 s4, s2
-; GFX1132_DPP-NEXT: s_mov_b32 s5, s3
-; GFX1132_DPP-NEXT: buffer_atomic_sub_u32 v0, off, s[4:7], 0 glc
+; GFX1132_DPP-NEXT: .LBB8_2: ; %atomicrmw.start
+; GFX1132_DPP-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132_DPP-NEXT: v_mov_b32_e32 v6, v4
+; GFX1132_DPP-NEXT: v_subrev_nc_u32_e32 v5, s9, v6
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132_DPP-NEXT: v_mov_b32_e32 v4, v5
+; GFX1132_DPP-NEXT: v_mov_b32_e32 v5, v6
+; GFX1132_DPP-NEXT: buffer_atomic_cmpswap_b32 v[4:5], off, s[4:7], 0 glc
; GFX1132_DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132_DPP-NEXT: buffer_gl1_inv
; GFX1132_DPP-NEXT: buffer_gl0_inv
-; GFX1132_DPP-NEXT: .LBB8_2:
+; GFX1132_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX1132_DPP-NEXT: s_or_b32 s10, vcc_lo, s10
+; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s10
+; GFX1132_DPP-NEXT: s_cbranch_execnz .LBB8_2
+; GFX1132_DPP-NEXT: ; %bb.3: ; %Flow
+; GFX1132_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s10
+; GFX1132_DPP-NEXT: .LBB8_4: ; %Flow3
+; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1132_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132_DPP-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132_DPP-NEXT: v_readfirstlane_b32 s2, v4
; GFX1132_DPP-NEXT: v_mov_b32_e32 v0, v3
; GFX1132_DPP-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132_DPP-NEXT: v_sub_nc_u32_e32 v0, s2, v0
-; GFX1132_DPP-NEXT: s_mov_b32 s2, s6
+; GFX1132_DPP-NEXT: s_mov_b32 s2, -1
; GFX1132_DPP-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX1132_DPP-NEXT: s_endpgm
;
@@ -5100,7 +5538,7 @@ define amdgpu_kernel void @sub_i32_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1232_DPP-NEXT: s_endpgm
entry:
%lane = call i32 @llvm.amdgcn.workitem.id.x()
- %old = atomicrmw sub ptr addrspace(1) %inout, i32 %lane syncscope("agent") acq_rel, !amdgpu.no.fine.grained.memory !0
+ %old = atomicrmw sub ptr addrspace(1) %inout, i32 %lane syncscope("agent") acq_rel
store i32 %old, ptr addrspace(1) %out
ret void
}
@@ -5110,35 +5548,57 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX7LESS: ; %bb.0: ; %entry
; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7LESS-NEXT: s_mov_b32 s4, 0
; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s7, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, s7, v0
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX7LESS-NEXT: s_cbranch_execz .LBB9_2
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX7LESS-NEXT: s_cbranch_execz .LBB9_4
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_mov_b32 s11, 0xf000
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX7LESS-NEXT: s_mul_i32 s6, s6, 5
-; GFX7LESS-NEXT: s_mov_b32 s10, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mov_b32 s8, s2
-; GFX7LESS-NEXT: s_mov_b32 s9, s3
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[8:11], 0 glc
+; GFX7LESS-NEXT: s_load_dwordx2 s[14:15], s[2:3], 0x0
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s5, s[6:7]
+; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, s4
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_mul_i32 s12, s5, 5
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s14
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s15
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s4, s2
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
+; GFX7LESS-NEXT: .LBB9_2: ; %atomicrmw.start
+; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: v_mov_b32_e32 v9, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v8, v0
+; GFX7LESS-NEXT: v_subrev_i32_e32 v6, vcc, s12, v8
+; GFX7LESS-NEXT: v_subb_u32_e32 v7, vcc, v9, v5, vcc
+; GFX7LESS-NEXT: s_waitcnt expcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, v6
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v7
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v8
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v9
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: buffer_wbinvl1
-; GFX7LESS-NEXT: .LBB9_2:
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX7LESS-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX7LESS-NEXT: s_cbranch_execnz .LBB9_2
+; GFX7LESS-NEXT: ; %bb.3: ; %Flow
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX7LESS-NEXT: .LBB9_4: ; %Flow3
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v1
; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v0
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v2
-; GFX7LESS-NEXT: v_mul_u32_u24_e32 v0, 5, v2
+; GFX7LESS-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v4
+; GFX7LESS-NEXT: v_mul_u32_u24_e32 v0, 5, v4
; GFX7LESS-NEXT: v_mov_b32_e32 v2, s4
; GFX7LESS-NEXT: v_sub_i32_e32 v0, vcc, s5, v0
; GFX7LESS-NEXT: v_subb_u32_e32 v1, vcc, v2, v1, vcc
@@ -5150,30 +5610,51 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_mov_b64 s[6:7], exec
; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX8-NEXT: s_mov_b32 s4, 0
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX8-NEXT: s_cbranch_execz .LBB9_2
+; GFX8-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX8-NEXT: s_cbranch_execz .LBB9_4
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mov_b32 s8, s2
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX8-NEXT: s_mul_i32 s2, s2, 5
-; GFX8-NEXT: s_mov_b32 s11, 0xf000
-; GFX8-NEXT: s_mov_b32 s10, -1
-; GFX8-NEXT: s_mov_b32 s9, s3
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[8:11], 0 glc
+; GFX8-NEXT: s_load_dwordx2 s[14:15], s[2:3], 0x0
+; GFX8-NEXT: s_bcnt1_i32_b64 s5, s[6:7]
+; GFX8-NEXT: s_mov_b64 s[10:11], 0
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
+; GFX8-NEXT: s_mul_i32 s12, s5, 5
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v0, s14
+; GFX8-NEXT: v_mov_b32_e32 v1, s15
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_mov_b32 s4, s2
+; GFX8-NEXT: s_mov_b32 s5, s3
+; GFX8-NEXT: .LBB9_2: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_subrev_u32_e32 v6, vcc, s12, v8
+; GFX8-NEXT: v_subb_u32_e32 v7, vcc, v9, v5, vcc
+; GFX8-NEXT: v_mov_b32_e32 v0, v6
+; GFX8-NEXT: v_mov_b32_e32 v1, v7
+; GFX8-NEXT: v_mov_b32_e32 v2, v8
+; GFX8-NEXT: v_mov_b32_e32 v3, v9
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: .LBB9_2:
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX8-NEXT: s_cbranch_execnz .LBB9_2
+; GFX8-NEXT: ; %bb.3: ; %Flow
+; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX8-NEXT: .LBB9_4: ; %Flow3
+; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: v_readfirstlane_b32 s4, v1
; GFX8-NEXT: v_readfirstlane_b32 s5, v0
-; GFX8-NEXT: v_mul_u32_u24_e32 v0, 5, v2
-; GFX8-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v2
+; GFX8-NEXT: v_mul_u32_u24_e32 v0, 5, v4
+; GFX8-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v4
; GFX8-NEXT: v_mov_b32_e32 v2, s4
; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s5, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
@@ -5188,30 +5669,51 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[6:7], exec
; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX9-NEXT: s_mov_b32 s4, 0
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX9-NEXT: s_cbranch_execz .LBB9_2
+; GFX9-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX9-NEXT: s_cbranch_execz .LBB9_4
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s8, s2
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX9-NEXT: s_mul_i32 s2, s2, 5
-; GFX9-NEXT: s_mov_b32 s11, 0xf000
-; GFX9-NEXT: s_mov_b32 s10, -1
-; GFX9-NEXT: s_mov_b32 s9, s3
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[8:11], 0 glc
+; GFX9-NEXT: s_load_dwordx2 s[14:15], s[2:3], 0x0
+; GFX9-NEXT: s_bcnt1_i32_b64 s5, s[6:7]
+; GFX9-NEXT: s_mov_b64 s[10:11], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s4
+; GFX9-NEXT: s_mul_i32 s12, s5, 5
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, s14
+; GFX9-NEXT: v_mov_b32_e32 v1, s15
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_mov_b32 s4, s2
+; GFX9-NEXT: s_mov_b32 s5, s3
+; GFX9-NEXT: .LBB9_2: ; %atomicrmw.start
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: v_mov_b32_e32 v9, v1
+; GFX9-NEXT: v_mov_b32_e32 v8, v0
+; GFX9-NEXT: v_subrev_co_u32_e32 v6, vcc, s12, v8
+; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, v9, v5, vcc
+; GFX9-NEXT: v_mov_b32_e32 v0, v6
+; GFX9-NEXT: v_mov_b32_e32 v1, v7
+; GFX9-NEXT: v_mov_b32_e32 v2, v8
+; GFX9-NEXT: v_mov_b32_e32 v3, v9
+; GFX9-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: .LBB9_2:
-; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX9-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX9-NEXT: s_cbranch_execnz .LBB9_2
+; GFX9-NEXT: ; %bb.3: ; %Flow
+; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX9-NEXT: .LBB9_4: ; %Flow3
+; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX9-NEXT: v_readfirstlane_b32 s4, v1
; GFX9-NEXT: v_readfirstlane_b32 s5, v0
-; GFX9-NEXT: v_mul_u32_u24_e32 v0, 5, v2
-; GFX9-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v2
+; GFX9-NEXT: v_mul_u32_u24_e32 v0, 5, v4
+; GFX9-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v4
; GFX9-NEXT: v_mov_b32_e32 v2, s4
; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, s5, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -5226,33 +5728,51 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064-NEXT: s_mov_b64 s[6:7], exec
; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
-; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX1064-NEXT: s_cbranch_execz .LBB9_2
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX1064-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX1064-NEXT: s_cbranch_execz .LBB9_4
; GFX1064-NEXT: ; %bb.1:
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0
-; GFX1064-NEXT: s_mul_i32 s6, s6, 5
-; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1064-NEXT: v_mov_b32_e32 v0, s6
-; GFX1064-NEXT: s_mov_b32 s10, -1
+; GFX1064-NEXT: s_mov_b64 s[10:11], 0
+; GFX1064-NEXT: s_mul_i32 s12, s6, 5
+; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT: s_mov_b32 s6, -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_mov_b32 s8, s2
-; GFX1064-NEXT: s_mov_b32 s9, s3
-; GFX1064-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[8:11], 0 glc
+; GFX1064-NEXT: v_mov_b32_e32 v0, s4
+; GFX1064-NEXT: v_mov_b32_e32 v1, s5
+; GFX1064-NEXT: s_mov_b32 s4, s2
+; GFX1064-NEXT: s_mov_b32 s5, s3
+; GFX1064-NEXT: .LBB9_2: ; %atomicrmw.start
+; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: v_mov_b32_e32 v8, v1
+; GFX1064-NEXT: v_mov_b32_e32 v7, v0
+; GFX1064-NEXT: v_sub_co_u32 v5, vcc, v7, s12
+; GFX1064-NEXT: v_subrev_co_ci_u32_e32 v6, vcc, 0, v8, vcc
+; GFX1064-NEXT: v_mov_b32_e32 v0, v5
+; GFX1064-NEXT: v_mov_b32_e32 v2, v7
+; GFX1064-NEXT: v_mov_b32_e32 v1, v6
+; GFX1064-NEXT: v_mov_b32_e32 v3, v8
+; GFX1064-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: buffer_gl1_inv
; GFX1064-NEXT: buffer_gl0_inv
-; GFX1064-NEXT: .LBB9_2:
-; GFX1064-NEXT: s_waitcnt_depctr 0xffe3
-; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GFX1064-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX1064-NEXT: s_cbranch_execnz .LBB9_2
+; GFX1064-NEXT: ; %bb.3: ; %Flow
+; GFX1064-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1064-NEXT: .LBB9_4: ; %Flow3
+; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1064-NEXT: v_mul_u32_u24_e32 v0, 5, v2
+; GFX1064-NEXT: v_mul_u32_u24_e32 v0, 5, v4
; GFX1064-NEXT: v_readfirstlane_b32 s3, v1
-; GFX1064-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v2
+; GFX1064-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v4
; GFX1064-NEXT: v_sub_co_u32 v0, vcc, s2, v0
; GFX1064-NEXT: v_sub_co_ci_u32_e32 v1, vcc, s3, v1, vcc
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
@@ -5264,33 +5784,51 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1032: ; %bb.0: ; %entry
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: s_mov_b32 s9, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
-; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo
-; GFX1032-NEXT: s_cbranch_execz .LBB9_2
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
+; GFX1032-NEXT: s_and_saveexec_b32 s8, vcc_lo
+; GFX1032-NEXT: s_cbranch_execz .LBB9_4
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1032-NEXT: v_mov_b32_e32 v1, 0
-; GFX1032-NEXT: s_mul_i32 s5, s5, 5
-; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1032-NEXT: v_mov_b32_e32 v0, s5
-; GFX1032-NEXT: s_mov_b32 s10, -1
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_mov_b32 s8, s2
-; GFX1032-NEXT: s_mov_b32 s9, s3
-; GFX1032-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[8:11], 0 glc
+; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: s_mul_i32 s10, s6, 5
+; GFX1032-NEXT: s_mov_b32 s6, -1
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v0, s4
+; GFX1032-NEXT: v_mov_b32_e32 v1, s5
+; GFX1032-NEXT: s_mov_b32 s4, s2
+; GFX1032-NEXT: s_mov_b32 s5, s3
+; GFX1032-NEXT: .LBB9_2: ; %atomicrmw.start
+; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: v_mov_b32_e32 v8, v1
+; GFX1032-NEXT: v_mov_b32_e32 v7, v0
+; GFX1032-NEXT: v_sub_co_u32 v5, vcc_lo, v7, s10
+; GFX1032-NEXT: v_subrev_co_ci_u32_e32 v6, vcc_lo, 0, v8, vcc_lo
+; GFX1032-NEXT: v_mov_b32_e32 v0, v5
+; GFX1032-NEXT: v_mov_b32_e32 v2, v7
+; GFX1032-NEXT: v_mov_b32_e32 v1, v6
+; GFX1032-NEXT: v_mov_b32_e32 v3, v8
+; GFX1032-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: buffer_gl1_inv
; GFX1032-NEXT: buffer_gl0_inv
-; GFX1032-NEXT: .LBB9_2:
-; GFX1032-NEXT: s_waitcnt_depctr 0xffe3
-; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[7:8]
+; GFX1032-NEXT: s_or_b32 s9, vcc_lo, s9
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s9
+; GFX1032-NEXT: s_cbranch_execnz .LBB9_2
+; GFX1032-NEXT: ; %bb.3: ; %Flow
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1032-NEXT: .LBB9_4: ; %Flow3
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1032-NEXT: v_mul_u32_u24_e32 v0, 5, v2
+; GFX1032-NEXT: v_mul_u32_u24_e32 v0, 5, v4
; GFX1032-NEXT: v_readfirstlane_b32 s3, v1
-; GFX1032-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v2
+; GFX1032-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v4
; GFX1032-NEXT: v_sub_co_u32 v0, vcc_lo, s2, v0
; GFX1032-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, s3, v1, vcc_lo
; GFX1032-NEXT: s_mov_b32 s3, 0x31016000
@@ -5302,34 +5840,58 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1164: ; %bb.0: ; %entry
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164-NEXT: s_mov_b64 s[6:7], exec
-; GFX1164-NEXT: s_mov_b64 s[4:5], exec
+; GFX1164-NEXT: s_mov_b64 s[8:9], exec
; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, s7, v0
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
; GFX1164-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
-; GFX1164-NEXT: s_cbranch_execz .LBB9_2
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-NEXT: s_cbranch_execz .LBB9_4
; GFX1164-NEXT: ; %bb.1:
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
-; GFX1164-NEXT: s_mul_i32 s6, s6, 5
-; GFX1164-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1164-NEXT: v_mov_b32_e32 v0, s6
-; GFX1164-NEXT: s_mov_b32 s10, -1
+; GFX1164-NEXT: s_mov_b64 s[10:11], 0
+; GFX1164-NEXT: s_mul_i32 s12, s6, 5
+; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-NEXT: s_mov_b32 s6, -1
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_mov_b32 s8, s2
-; GFX1164-NEXT: s_mov_b32 s9, s3
-; GFX1164-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[8:11], 0 glc
+; GFX1164-NEXT: v_mov_b32_e32 v0, s4
+; GFX1164-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-NEXT: s_mov_b32 s4, s2
+; GFX1164-NEXT: s_mov_b32 s5, s3
+; GFX1164-NEXT: .LBB9_2: ; %atomicrmw.start
+; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_mov_b32_e32 v8, v1
+; GFX1164-NEXT: v_mov_b32_e32 v7, v0
+; GFX1164-NEXT: v_sub_co_u32 v5, vcc, v7, s12
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_subrev_co_ci_u32_e64 v6, null, 0, v8, vcc
+; GFX1164-NEXT: v_mov_b32_e32 v0, v5
+; GFX1164-NEXT: v_mov_b32_e32 v2, v7
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-NEXT: v_mov_b32_e32 v1, v6
+; GFX1164-NEXT: v_mov_b32_e32 v3, v8
+; GFX1164-NEXT: buffer_atomic_cmpswap_b64 v[0:3], off, s[4:7], 0 glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: buffer_gl1_inv
; GFX1164-NEXT: buffer_gl0_inv
-; GFX1164-NEXT: .LBB9_2:
-; GFX1164-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GFX1164-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1164-NEXT: s_cbranch_execnz .LBB9_2
+; GFX1164-NEXT: ; %bb.3: ; %Flow
+; GFX1164-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1164-NEXT: .LBB9_4: ; %Flow3
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1164-NEXT: v_mul_u32_u24_e32 v0, 5, v2
+; GFX1164-NEXT: v_mul_u32_u24_e32 v0, 5, v4
; GFX1164-NEXT: v_readfirstlane_b32 s3, v1
-; GFX1164-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v2
+; GFX1164-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v4
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-NEXT: v_sub_co_u32 v0, vcc, s2, v0
; GFX1164-NEXT: v_sub_co_ci_u32_e64 v1, null, s3, v1, vcc
@@ -5342,32 +5904,54 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1132: ; %bb.0: ; %entry
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132-NEXT: s_mov_b32 s6, exec_lo
-; GFX1132-NEXT: s_mov_b32 s4, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s6, 0
+; GFX1132-NEXT: s_mov_b32 s9, 0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1132-NEXT: s_mov_b32 s8, exec_lo
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
-; GFX1132-NEXT: s_cbranch_execz .LBB9_2
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-NEXT: s_cbranch_execz .LBB9_4
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s5, s6
-; GFX1132-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1132-NEXT: s_mul_i32 s5, s5, 5
-; GFX1132-NEXT: s_mov_b32 s10, -1
-; GFX1132-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v1, 0
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_mov_b32 s8, s2
-; GFX1132-NEXT: s_mov_b32 s9, s3
-; GFX1132-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[8:11], 0 glc
+; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-NEXT: s_mul_i32 s10, s6, 5
+; GFX1132-NEXT: s_mov_b32 s6, -1
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
+; GFX1132-NEXT: s_mov_b32 s4, s2
+; GFX1132-NEXT: s_mov_b32 s5, s3
+; GFX1132-NEXT: .LBB9_2: ; %atomicrmw.start
+; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
+; GFX1132-NEXT: v_sub_co_u32 v5, vcc_lo, v7, s10
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-NEXT: v_subrev_co_ci_u32_e64 v6, null, 0, v8, vcc_lo
+; GFX1132-NEXT: v_mov_b32_e32 v0, v5
+; GFX1132-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1132-NEXT: v_mov_b32_e32 v1, v6
+; GFX1132-NEXT: buffer_atomic_cmpswap_b64 v[0:3], off, s[4:7], 0 glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: buffer_gl1_inv
; GFX1132-NEXT: buffer_gl0_inv
-; GFX1132-NEXT: .LBB9_2:
-; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[7:8]
+; GFX1132-NEXT: s_or_b32 s9, vcc_lo, s9
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s9
+; GFX1132-NEXT: s_cbranch_execnz .LBB9_2
+; GFX1132-NEXT: ; %bb.3: ; %Flow
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s9
+; GFX1132-NEXT: .LBB9_4: ; %Flow3
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s8
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1132-NEXT: v_mul_u32_u24_e32 v0, 5, v2
+; GFX1132-NEXT: v_mul_u32_u24_e32 v0, 5, v4
; GFX1132-NEXT: v_readfirstlane_b32 s3, v1
-; GFX1132-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v2
+; GFX1132-NEXT: v_mul_hi_u32_u24_e32 v1, 5, v4
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-NEXT: v_sub_co_u32 v0, vcc_lo, s2, v0
; GFX1132-NEXT: v_sub_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
@@ -5454,7 +6038,7 @@ define amdgpu_kernel void @sub_i64_constant(ptr addrspace(1) %out, ptr addrspace
; GFX1232-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null
; GFX1232-NEXT: s_endpgm
entry:
- %old = atomicrmw sub ptr addrspace(1) %inout, i64 5 syncscope("agent") acq_rel, !amdgpu.no.fine.grained.memory !0
+ %old = atomicrmw sub ptr addrspace(1) %inout, i64 5 syncscope("agent") acq_rel
store i64 %old, ptr addrspace(1) %out
ret void
}
@@ -5462,45 +6046,67 @@ entry:
define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(1) %inout, i64 %subitive) {
; GFX7LESS-LABEL: sub_i64_uniform:
; GFX7LESS: ; %bb.0: ; %entry
-; GFX7LESS-NEXT: s_mov_b64 s[8:9], exec
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s8, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s9, v0
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX7LESS-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v4, s7, v0
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX7LESS-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GFX7LESS-NEXT: s_cbranch_execz .LBB10_2
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[10:11], vcc
+; GFX7LESS-NEXT: s_cbranch_execz .LBB10_4
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_mov_b32 s15, 0xf000
-; GFX7LESS-NEXT: s_mov_b32 s14, -1
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mov_b32 s12, s2
-; GFX7LESS-NEXT: s_mov_b32 s13, s3
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[8:9]
-; GFX7LESS-NEXT: s_mul_i32 s3, s5, s2
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
-; GFX7LESS-NEXT: v_mul_hi_u32 v0, s4, v0
-; GFX7LESS-NEXT: s_mul_i32 s2, s4, s2
-; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, s3, v0
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
-; GFX7LESS-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[12:15], 0 glc
+; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
+; GFX7LESS-NEXT: s_mov_b64 s[12:13], 0
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_mul_i32 s14, s9, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
+; GFX7LESS-NEXT: v_mul_hi_u32 v0, s8, v0
+; GFX7LESS-NEXT: s_mul_i32 s6, s8, s6
+; GFX7LESS-NEXT: v_add_i32_e32 v5, vcc, s14, v0
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, s6
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s4, s2
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
+; GFX7LESS-NEXT: .LBB10_2: ; %atomicrmw.start
+; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: v_mov_b32_e32 v10, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v9, v0
+; GFX7LESS-NEXT: v_sub_i32_e32 v7, vcc, v9, v6
+; GFX7LESS-NEXT: v_subb_u32_e32 v8, vcc, v10, v5, vcc
+; GFX7LESS-NEXT: s_waitcnt expcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, v7
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v8
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v9
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v10
+; GFX7LESS-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: buffer_wbinvl1
-; GFX7LESS-NEXT: .LBB10_2:
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX7LESS-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[12:13]
+; GFX7LESS-NEXT: s_cbranch_execnz .LBB10_2
+; GFX7LESS-NEXT: ; %bb.3: ; %Flow
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX7LESS-NEXT: .LBB10_4: ; %Flow4
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: v_readfirstlane_b32 s6, v1
-; GFX7LESS-NEXT: v_readfirstlane_b32 s7, v0
+; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v1
+; GFX7LESS-NEXT: v_readfirstlane_b32 s5, v0
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mul_lo_u32 v0, s5, v2
-; GFX7LESS-NEXT: v_mul_hi_u32 v1, s4, v2
-; GFX7LESS-NEXT: v_mul_lo_u32 v2, s4, v2
+; GFX7LESS-NEXT: v_mul_lo_u32 v0, s9, v4
+; GFX7LESS-NEXT: v_mul_hi_u32 v1, s8, v4
+; GFX7LESS-NEXT: v_mul_lo_u32 v2, s8, v4
; GFX7LESS-NEXT: v_add_i32_e32 v1, vcc, v1, v0
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, s6
-; GFX7LESS-NEXT: v_sub_i32_e32 v0, vcc, s7, v2
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s4
+; GFX7LESS-NEXT: v_sub_i32_e32 v0, vcc, s5, v2
; GFX7LESS-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
; GFX7LESS-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX7LESS-NEXT: s_endpgm
@@ -5508,33 +6114,54 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX8-LABEL: sub_i64_uniform:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
-; GFX8-NEXT: s_mov_b64 s[8:9], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX8-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34
+; GFX8-NEXT: s_mov_b64 s[6:7], exec
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v6, s7, v0
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v6
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX8-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GFX8-NEXT: s_cbranch_execz .LBB10_2
+; GFX8-NEXT: s_and_saveexec_b64 s[10:11], vcc
+; GFX8-NEXT: s_cbranch_execz .LBB10_4
; GFX8-NEXT: ; %bb.1:
+; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mov_b32 s12, s2
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[8:9]
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mad_u64_u32 v[0:1], s[8:9], s4, v0, 0
-; GFX8-NEXT: s_mul_i32 s2, s5, s2
-; GFX8-NEXT: s_mov_b32 s15, 0xf000
-; GFX8-NEXT: s_mov_b32 s14, -1
-; GFX8-NEXT: s_mov_b32 s13, s3
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, s2, v1
-; GFX8-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[12:15], 0 glc
+; GFX8-NEXT: v_mad_u64_u32 v[4:5], s[4:5], s8, v0, 0
+; GFX8-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
+; GFX8-NEXT: s_mul_i32 s6, s9, s6
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, s6, v5
+; GFX8-NEXT: s_mov_b64 s[12:13], 0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v0, s4
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_mov_b32 s4, s2
+; GFX8-NEXT: s_mov_b32 s5, s3
+; GFX8-NEXT: .LBB10_2: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_mov_b32_e32 v10, v1
+; GFX8-NEXT: v_mov_b32_e32 v9, v0
+; GFX8-NEXT: v_sub_u32_e32 v7, vcc, v9, v4
+; GFX8-NEXT: v_subb_u32_e32 v8, vcc, v10, v5, vcc
+; GFX8-NEXT: v_mov_b32_e32 v0, v7
+; GFX8-NEXT: v_mov_b32_e32 v1, v8
+; GFX8-NEXT: v_mov_b32_e32 v2, v9
+; GFX8-NEXT: v_mov_b32_e32 v3, v10
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: .LBB10_2:
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX8-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[12:13]
+; GFX8-NEXT: s_cbranch_execnz .LBB10_2
+; GFX8-NEXT: ; %bb.3: ; %Flow
+; GFX8-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX8-NEXT: .LBB10_4: ; %Flow4
+; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u32 v4, s5, v2
-; GFX8-NEXT: v_mad_u64_u32 v[2:3], s[4:5], s4, v2, 0
+; GFX8-NEXT: v_mul_lo_u32 v4, s9, v6
+; GFX8-NEXT: v_mad_u64_u32 v[2:3], s[4:5], s8, v6, 0
; GFX8-NEXT: v_readfirstlane_b32 s4, v1
; GFX8-NEXT: v_readfirstlane_b32 s5, v0
; GFX8-NEXT: v_add_u32_e32 v1, vcc, v3, v4
@@ -5549,43 +6176,63 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX9-LABEL: sub_i64_uniform:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX9-NEXT: s_mov_b64 s[8:9], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
+; GFX9-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34
+; GFX9-NEXT: s_mov_b64 s[6:7], exec
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
; GFX9-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX9-NEXT: s_cbranch_execz .LBB10_2
+; GFX9-NEXT: s_and_saveexec_b64 s[10:11], vcc
+; GFX9-NEXT: s_cbranch_execz .LBB10_4
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s12, s2
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[8:9]
-; GFX9-NEXT: s_mov_b32 s13, s3
-; GFX9-NEXT: s_mul_i32 s3, s7, s2
-; GFX9-NEXT: s_mul_hi_u32 s8, s6, s2
-; GFX9-NEXT: s_add_i32 s8, s8, s3
-; GFX9-NEXT: s_mul_i32 s2, s6, s2
-; GFX9-NEXT: s_mov_b32 s15, 0xf000
-; GFX9-NEXT: s_mov_b32 s14, -1
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s8
-; GFX9-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[12:15], 0 glc
+; GFX9-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
+; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX9-NEXT: s_mul_i32 s7, s9, s6
+; GFX9-NEXT: s_mul_hi_u32 s12, s8, s6
+; GFX9-NEXT: s_add_i32 s7, s12, s7
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-NEXT: s_mul_i32 s14, s8, s6
+; GFX9-NEXT: s_mov_b64 s[12:13], 0
+; GFX9-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-NEXT: v_mov_b32_e32 v5, s7
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_mov_b32 s4, s2
+; GFX9-NEXT: s_mov_b32 s5, s3
+; GFX9-NEXT: .LBB10_2: ; %atomicrmw.start
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: v_mov_b32_e32 v9, v1
+; GFX9-NEXT: v_mov_b32_e32 v8, v0
+; GFX9-NEXT: v_subrev_co_u32_e32 v6, vcc, s14, v8
+; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, v9, v5, vcc
+; GFX9-NEXT: v_mov_b32_e32 v0, v6
+; GFX9-NEXT: v_mov_b32_e32 v1, v7
+; GFX9-NEXT: v_mov_b32_e32 v2, v8
+; GFX9-NEXT: v_mov_b32_e32 v3, v9
+; GFX9-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: .LBB10_2:
-; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX9-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[12:13]
+; GFX9-NEXT: s_cbranch_execnz .LBB10_2
+; GFX9-NEXT: ; %bb.3: ; %Flow
+; GFX9-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX9-NEXT: .LBB10_4: ; %Flow4
+; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mad_u64_u32 v[3:4], s[4:5], s6, v2, 0
-; GFX9-NEXT: v_readfirstlane_b32 s8, v0
+; GFX9-NEXT: v_mad_u64_u32 v[2:3], s[4:5], s8, v4, 0
+; GFX9-NEXT: v_readfirstlane_b32 s7, v0
; GFX9-NEXT: v_readfirstlane_b32 s6, v1
-; GFX9-NEXT: v_mov_b32_e32 v0, v4
-; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s7, v2, v[0:1]
-; GFX9-NEXT: v_mov_b32_e32 v2, s6
-; GFX9-NEXT: v_sub_co_u32_e32 v1, vcc, s8, v3
+; GFX9-NEXT: v_mov_b32_e32 v0, v3
+; GFX9-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s9, v4, v[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v3, s6
+; GFX9-NEXT: v_sub_co_u32_e32 v1, vcc, s7, v2
; GFX9-NEXT: s_mov_b32 s3, 0xf000
; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v0, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v3, v0, vcc
; GFX9-NEXT: buffer_store_dwordx2 v[1:2], off, s[0:3], 0
; GFX9-NEXT: s_endpgm
;
@@ -5593,41 +6240,59 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1064: ; %bb.0: ; %entry
; GFX1064-NEXT: s_clause 0x1
; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX1064-NEXT: s_mov_b64 s[8:9], exec
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v0
+; GFX1064-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34
+; GFX1064-NEXT: s_mov_b64 s[6:7], exec
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
-; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX1064-NEXT: s_cbranch_execz .LBB10_2
+; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v4
+; GFX1064-NEXT: s_and_saveexec_b64 s[10:11], vcc
+; GFX1064-NEXT: s_cbranch_execz .LBB10_4
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s8, s[8:9]
-; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_mul_i32 s9, s7, s8
-; GFX1064-NEXT: s_mul_hi_u32 s10, s6, s8
-; GFX1064-NEXT: s_mul_i32 s8, s6, s8
-; GFX1064-NEXT: s_add_i32 s10, s10, s9
-; GFX1064-NEXT: v_mov_b32_e32 v0, s8
-; GFX1064-NEXT: v_mov_b32_e32 v1, s10
-; GFX1064-NEXT: s_mov_b32 s10, -1
-; GFX1064-NEXT: s_mov_b32 s8, s2
-; GFX1064-NEXT: s_mov_b32 s9, s3
-; GFX1064-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[8:11], 0 glc
+; GFX1064-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
+; GFX1064-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1064-NEXT: s_mov_b64 s[12:13], 0
+; GFX1064-NEXT: s_mul_i32 s7, s9, s6
+; GFX1064-NEXT: s_mul_hi_u32 s15, s8, s6
+; GFX1064-NEXT: s_mul_i32 s14, s8, s6
+; GFX1064-NEXT: s_add_i32 s15, s15, s7
+; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT: s_mov_b32 s6, -1
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: v_mov_b32_e32 v0, s4
+; GFX1064-NEXT: v_mov_b32_e32 v1, s5
+; GFX1064-NEXT: s_mov_b32 s4, s2
+; GFX1064-NEXT: s_mov_b32 s5, s3
+; GFX1064-NEXT: .LBB10_2: ; %atomicrmw.start
+; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: v_mov_b32_e32 v8, v1
+; GFX1064-NEXT: v_mov_b32_e32 v7, v0
+; GFX1064-NEXT: v_sub_co_u32 v5, vcc, v7, s14
+; GFX1064-NEXT: v_subrev_co_ci_u32_e32 v6, vcc, s15, v8, vcc
+; GFX1064-NEXT: v_mov_b32_e32 v0, v5
+; GFX1064-NEXT: v_mov_b32_e32 v2, v7
+; GFX1064-NEXT: v_mov_b32_e32 v1, v6
+; GFX1064-NEXT: v_mov_b32_e32 v3, v8
+; GFX1064-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
; GFX1064-NEXT: buffer_gl1_inv
; GFX1064-NEXT: buffer_gl0_inv
-; GFX1064-NEXT: .LBB10_2:
-; GFX1064-NEXT: s_waitcnt_depctr 0xffe3
-; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GFX1064-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[12:13]
+; GFX1064-NEXT: s_cbranch_execnz .LBB10_2
+; GFX1064-NEXT: ; %bb.3: ; %Flow
+; GFX1064-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1064-NEXT: .LBB10_4: ; %Flow4
+; GFX1064-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mad_u64_u32 v[3:4], s[2:3], s6, v2, 0
-; GFX1064-NEXT: v_mad_u64_u32 v[4:5], s[2:3], s7, v2, v[4:5]
+; GFX1064-NEXT: v_mad_u64_u32 v[2:3], s[2:3], s8, v4, 0
+; GFX1064-NEXT: v_mad_u64_u32 v[3:4], s[2:3], s9, v4, v[3:4]
; GFX1064-NEXT: v_readfirstlane_b32 s2, v0
; GFX1064-NEXT: v_readfirstlane_b32 s3, v1
-; GFX1064-NEXT: v_sub_co_u32 v0, vcc, s2, v3
-; GFX1064-NEXT: v_sub_co_ci_u32_e32 v1, vcc, s3, v4, vcc
+; GFX1064-NEXT: v_sub_co_u32 v0, vcc, s2, v2
+; GFX1064-NEXT: v_sub_co_ci_u32_e32 v1, vcc, s3, v3, vcc
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
; GFX1064-NEXT: s_mov_b32 s2, -1
; GFX1064-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -5637,40 +6302,58 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1032: ; %bb.0: ; %entry
; GFX1032-NEXT: s_clause 0x1
; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX1032-NEXT: s_mov_b32 s8, exec_lo
+; GFX1032-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34
+; GFX1032-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032-NEXT: s_mov_b32 s11, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
; GFX1032-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v2, s8, 0
-; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
-; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo
-; GFX1032-NEXT: s_cbranch_execz .LBB10_2
+; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v4
+; GFX1032-NEXT: s_and_saveexec_b32 s10, vcc_lo
+; GFX1032-NEXT: s_cbranch_execz .LBB10_4
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s5, s8
-; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_mul_i32 s8, s7, s5
-; GFX1032-NEXT: s_mul_hi_u32 s9, s6, s5
-; GFX1032-NEXT: s_mul_i32 s5, s6, s5
-; GFX1032-NEXT: s_add_i32 s9, s9, s8
-; GFX1032-NEXT: v_mov_b32_e32 v0, s5
-; GFX1032-NEXT: v_mov_b32_e32 v1, s9
-; GFX1032-NEXT: s_mov_b32 s10, -1
-; GFX1032-NEXT: s_mov_b32 s8, s2
-; GFX1032-NEXT: s_mov_b32 s9, s3
-; GFX1032-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[8:11], 0 glc
+; GFX1032-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
+; GFX1032-NEXT: s_bcnt1_i32_b32 s6, s6
+; GFX1032-NEXT: s_mul_i32 s7, s9, s6
+; GFX1032-NEXT: s_mul_hi_u32 s13, s8, s6
+; GFX1032-NEXT: s_mul_i32 s12, s8, s6
+; GFX1032-NEXT: s_add_i32 s13, s13, s7
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: s_mov_b32 s6, -1
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v0, s4
+; GFX1032-NEXT: v_mov_b32_e32 v1, s5
+; GFX1032-NEXT: s_mov_b32 s4, s2
+; GFX1032-NEXT: s_mov_b32 s5, s3
+; GFX1032-NEXT: .LBB10_2: ; %atomicrmw.start
+; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: v_mov_b32_e32 v8, v1
+; GFX1032-NEXT: v_mov_b32_e32 v7, v0
+; GFX1032-NEXT: v_sub_co_u32 v5, vcc_lo, v7, s12
+; GFX1032-NEXT: v_subrev_co_ci_u32_e32 v6, vcc_lo, s13, v8, vcc_lo
+; GFX1032-NEXT: v_mov_b32_e32 v0, v5
+; GFX1032-NEXT: v_mov_b32_e32 v2, v7
+; GFX1032-NEXT: v_mov_b32_e32 v1, v6
+; GFX1032-NEXT: v_mov_b32_e32 v3, v8
+; GFX1032-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
; GFX1032-NEXT: buffer_gl1_inv
; GFX1032-NEXT: buffer_gl0_inv
-; GFX1032-NEXT: .LBB10_2:
-; GFX1032-NEXT: s_waitcnt_depctr 0xffe3
-; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[7:8]
+; GFX1032-NEXT: s_or_b32 s11, vcc_lo, s11
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s11
+; GFX1032-NEXT: s_cbranch_execnz .LBB10_2
+; GFX1032-NEXT: ; %bb.3: ; %Flow
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s11
+; GFX1032-NEXT: .LBB10_4: ; %Flow4
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s10
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mad_u64_u32 v[3:4], s2, s6, v2, 0
+; GFX1032-NEXT: v_mad_u64_u32 v[2:3], s2, s8, v4, 0
; GFX1032-NEXT: v_readfirstlane_b32 s3, v1
-; GFX1032-NEXT: v_mad_u64_u32 v[4:5], s2, s7, v2, v[4:5]
+; GFX1032-NEXT: v_mad_u64_u32 v[3:4], s2, s9, v4, v[3:4]
; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1032-NEXT: v_sub_co_u32 v0, vcc_lo, s2, v3
-; GFX1032-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, s3, v4, vcc_lo
+; GFX1032-NEXT: v_sub_co_u32 v0, vcc_lo, s2, v2
+; GFX1032-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, s3, v3, vcc_lo
; GFX1032-NEXT: s_mov_b32 s3, 0x31016000
; GFX1032-NEXT: s_mov_b32 s2, -1
; GFX1032-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -5680,41 +6363,65 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1164: ; %bb.0: ; %entry
; GFX1164-NEXT: s_clause 0x1
; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1164-NEXT: s_mov_b64 s[8:9], exec
+; GFX1164-NEXT: s_load_b64 s[8:9], s[4:5], 0x34
; GFX1164-NEXT: s_mov_b64 s[6:7], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s8, 0
+; GFX1164-NEXT: s_mov_b64 s[10:11], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v2, s9, v0
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v4, s7, v0
; GFX1164-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v2
-; GFX1164-NEXT: s_cbranch_execz .LBB10_2
+; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1164-NEXT: s_cbranch_execz .LBB10_4
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s8, s[8:9]
-; GFX1164-NEXT: s_mov_b32 s11, 0x31016000
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_mul_i32 s9, s5, s8
-; GFX1164-NEXT: s_mul_hi_u32 s10, s4, s8
-; GFX1164-NEXT: s_mul_i32 s8, s4, s8
-; GFX1164-NEXT: s_add_i32 s10, s10, s9
-; GFX1164-NEXT: v_mov_b32_e32 v0, s8
-; GFX1164-NEXT: v_mov_b32_e32 v1, s10
-; GFX1164-NEXT: s_mov_b32 s10, -1
-; GFX1164-NEXT: s_mov_b32 s8, s2
-; GFX1164-NEXT: s_mov_b32 s9, s3
-; GFX1164-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[8:11], 0 glc
+; GFX1164-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
+; GFX1164-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX1164-NEXT: s_mov_b64 s[12:13], 0
+; GFX1164-NEXT: s_mul_i32 s7, s9, s6
+; GFX1164-NEXT: s_mul_hi_u32 s15, s8, s6
+; GFX1164-NEXT: s_mul_i32 s14, s8, s6
+; GFX1164-NEXT: s_add_i32 s15, s15, s7
+; GFX1164-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-NEXT: s_mov_b32 s6, -1
+; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-NEXT: v_mov_b32_e32 v0, s4
+; GFX1164-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164-NEXT: s_mov_b32 s4, s2
+; GFX1164-NEXT: s_mov_b32 s5, s3
+; GFX1164-NEXT: .LBB10_2: ; %atomicrmw.start
+; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164-NEXT: v_mov_b32_e32 v8, v1
+; GFX1164-NEXT: v_mov_b32_e32 v7, v0
+; GFX1164-NEXT: v_sub_co_u32 v5, vcc, v7, s14
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164-NEXT: v_subrev_co_ci_u32_e64 v6, null, s15, v8, vcc
+; GFX1164-NEXT: v_mov_b32_e32 v0, v5
+; GFX1164-NEXT: v_mov_b32_e32 v2, v7
+; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164-NEXT: v_mov_b32_e32 v1, v6
+; GFX1164-NEXT: v_mov_b32_e32 v3, v8
+; GFX1164-NEXT: buffer_atomic_cmpswap_b64 v[0:3], off, s[4:7], 0 glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: buffer_gl1_inv
; GFX1164-NEXT: buffer_gl0_inv
-; GFX1164-NEXT: .LBB10_2:
-; GFX1164-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GFX1164-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: s_and_not1_b64 exec, exec, s[12:13]
+; GFX1164-NEXT: s_cbranch_execnz .LBB10_2
+; GFX1164-NEXT: ; %bb.3: ; %Flow
+; GFX1164-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1164-NEXT: .LBB10_4: ; %Flow4
+; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mad_u64_u32 v[3:4], null, s4, v2, 0
+; GFX1164-NEXT: v_mad_u64_u32 v[2:3], null, s8, v4, 0
; GFX1164-NEXT: v_readfirstlane_b32 s2, v0
; GFX1164-NEXT: v_readfirstlane_b32 s3, v1
; GFX1164-NEXT: s_waitcnt_depctr 0xfff
-; GFX1164-NEXT: v_mad_u64_u32 v[5:6], null, s5, v2, v[4:5]
-; GFX1164-NEXT: v_sub_co_u32 v0, vcc, s2, v3
+; GFX1164-NEXT: v_mad_u64_u32 v[5:6], null, s9, v4, v[3:4]
+; GFX1164-NEXT: v_sub_co_u32 v0, vcc, s2, v2
; GFX1164-NEXT: s_mov_b32 s2, -1
; GFX1164-NEXT: v_sub_co_ci_u32_e64 v1, null, s3, v5, vcc
; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
@@ -5725,40 +6432,62 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1132: ; %bb.0: ; %entry
; GFX1132-NEXT: s_clause 0x1
; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1132-NEXT: s_mov_b32 s7, exec_lo
+; GFX1132-NEXT: s_load_b64 s[8:9], s[4:5], 0x34
; GFX1132-NEXT: s_mov_b32 s6, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0
+; GFX1132-NEXT: s_mov_b32 s11, 0
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v4, s6, 0
+; GFX1132-NEXT: s_mov_b32 s10, exec_lo
; GFX1132-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v2
-; GFX1132-NEXT: s_cbranch_execz .LBB10_2
+; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v4
+; GFX1132-NEXT: s_cbranch_execz .LBB10_4
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s7, s7
-; GFX1132-NEXT: s_mov_b32 s11, 0x31016000
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_mul_i32 s8, s5, s7
-; GFX1132-NEXT: s_mul_hi_u32 s9, s4, s7
-; GFX1132-NEXT: s_mul_i32 s7, s4, s7
-; GFX1132-NEXT: s_add_i32 s9, s9, s8
+; GFX1132-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
+; GFX1132-NEXT: s_bcnt1_i32_b32 s6, s6
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v0, s7 :: v_dual_mov_b32 v1, s9
-; GFX1132-NEXT: s_mov_b32 s10, -1
-; GFX1132-NEXT: s_mov_b32 s8, s2
-; GFX1132-NEXT: s_mov_b32 s9, s3
-; GFX1132-NEXT: buffer_atomic_sub_u64 v[0:1], off, s[8:11], 0 glc
+; GFX1132-NEXT: s_mul_i32 s7, s9, s6
+; GFX1132-NEXT: s_mul_hi_u32 s13, s8, s6
+; GFX1132-NEXT: s_mul_i32 s12, s8, s6
+; GFX1132-NEXT: s_add_i32 s13, s13, s7
+; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-NEXT: s_mov_b32 s6, -1
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
+; GFX1132-NEXT: s_mov_b32 s4, s2
+; GFX1132-NEXT: s_mov_b32 s5, s3
+; GFX1132-NEXT: .LBB10_2: ; %atomicrmw.start
+; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
+; GFX1132-NEXT: v_sub_co_u32 v5, vcc_lo, v7, s12
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132-NEXT: v_subrev_co_ci_u32_e64 v6, null, s13, v8, vcc_lo
+; GFX1132-NEXT: v_mov_b32_e32 v0, v5
+; GFX1132-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1132-NEXT: v_mov_b32_e32 v1, v6
+; GFX1132-NEXT: buffer_atomic_cmpswap_b64 v[0:3], off, s[4:7], 0 glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: buffer_gl1_inv
; GFX1132-NEXT: buffer_gl0_inv
-; GFX1132-NEXT: .LBB10_2:
-; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[7:8]
+; GFX1132-NEXT: s_or_b32 s11, vcc_lo, s11
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-NEXT: s_and_not1_b32 exec_lo, exec_lo, s11
+; GFX1132-NEXT: s_cbranch_execnz .LBB10_2
+; GFX1132-NEXT: ; %bb.3: ; %Flow
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s11
+; GFX1132-NEXT: .LBB10_4: ; %Flow4
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s10
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_mad_u64_u32 v[3:4], null, s4, v2, 0
+; GFX1132-NEXT: v_mad_u64_u32 v[2:3], null, s8, v4, 0
; GFX1132-NEXT: v_readfirstlane_b32 s2, v0
; GFX1132-NEXT: v_readfirstlane_b32 s3, v1
-; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1132-NEXT: v_mad_u64_u32 v[5:6], null, s5, v2, v[4:5]
-; GFX1132-NEXT: v_sub_co_u32 v0, vcc_lo, s2, v3
+; GFX1132-NEXT: v_mad_u64_u32 v[5:6], null, s9, v4, v[3:4]
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1132-NEXT: v_sub_co_u32 v0, vcc_lo, s2, v2
; GFX1132-NEXT: s_mov_b32 s2, -1
; GFX1132-NEXT: v_sub_co_ci_u32_e64 v1, null, s3, v5, vcc_lo
; GFX1132-NEXT: s_mov_b32 s3, 0x31016000
@@ -5850,7 +6579,7 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1232-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null
; GFX1232-NEXT: s_endpgm
entry:
- %old = atomicrmw sub ptr addrspace(1) %inout, i64 %subitive syncscope("agent") acq_rel, !amdgpu.no.fine.grained.memory !0
+ %old = atomicrmw sub ptr addrspace(1) %inout, i64 %subitive syncscope("agent") acq_rel
store i64 %old, ptr addrspace(1) %out
ret void
}
@@ -5859,19 +6588,19 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS_ITERATIVE-LABEL: sub_i64_varying:
; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
-; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[6:7], 0
-; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
+; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[8:9], 0
+; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX7LESS_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 m0, s2
-; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s3, v3, s2
-; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s8, v0, s2
-; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v2, s7, m0
-; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v1, s6, m0
-; GFX7LESS_ITERATIVE-NEXT: s_add_u32 s6, s6, s8
-; GFX7LESS_ITERATIVE-NEXT: s_addc_u32 s7, s7, s3
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s3, v1, s2
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s2
+; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v5, s9, m0
+; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v4, s8, m0
+; GFX7LESS_ITERATIVE-NEXT: s_add_u32 s8, s8, s6
+; GFX7LESS_ITERATIVE-NEXT: s_addc_u32 s9, s9, s3
; GFX7LESS_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
; GFX7LESS_ITERATIVE-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
@@ -5882,51 +6611,72 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
; GFX7LESS_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GFX7LESS_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr3_vgpr4
+; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX7LESS_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX7LESS_ITERATIVE-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GFX7LESS_ITERATIVE-NEXT: s_cbranch_execz .LBB11_4
+; GFX7LESS_ITERATIVE-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
+; GFX7LESS_ITERATIVE-NEXT: s_cbranch_execz .LBB11_6
; GFX7LESS_ITERATIVE-NEXT: ; %bb.3:
-; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s11, 0xf000
-; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s10, -1
; GFX7LESS_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s8, s2
-; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s9, s3
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v3, s6
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v4, s7
-; GFX7LESS_ITERATIVE-NEXT: buffer_atomic_sub_x2 v[3:4], off, s[8:11], 0 glc
+; GFX7LESS_ITERATIVE-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
+; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[12:13], 0
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v6, s9
+; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, s4
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, s5
+; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s6, -1
+; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s4, s2
+; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s5, s3
+; GFX7LESS_ITERATIVE-NEXT: .LBB11_4: ; %atomicrmw.start
+; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v10, v1
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v9, v0
+; GFX7LESS_ITERATIVE-NEXT: v_subrev_i32_e32 v7, vcc, s8, v9
+; GFX7LESS_ITERATIVE-NEXT: v_subb_u32_e32 v8, vcc, v10, v6, vcc
+; GFX7LESS_ITERATIVE-NEXT: s_waitcnt expcnt(0)
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, v7
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, v8
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v2, v9
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v3, v10
+; GFX7LESS_ITERATIVE-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX7LESS_ITERATIVE-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS_ITERATIVE-NEXT: buffer_wbinvl1
-; GFX7LESS_ITERATIVE-NEXT: .LBB11_4:
-; GFX7LESS_ITERATIVE-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX7LESS_ITERATIVE-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX7LESS_ITERATIVE-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
+; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 exec, exec, s[12:13]
+; GFX7LESS_ITERATIVE-NEXT: s_cbranch_execnz .LBB11_4
+; GFX7LESS_ITERATIVE-NEXT: ; %bb.5: ; %Flow
+; GFX7LESS_ITERATIVE-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX7LESS_ITERATIVE-NEXT: .LBB11_6: ; %Flow4
+; GFX7LESS_ITERATIVE-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX7LESS_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s2, -1
-; GFX7LESS_ITERATIVE-NEXT: v_readfirstlane_b32 s4, v4
-; GFX7LESS_ITERATIVE-NEXT: v_readfirstlane_b32 s5, v3
+; GFX7LESS_ITERATIVE-NEXT: v_readfirstlane_b32 s4, v1
+; GFX7LESS_ITERATIVE-NEXT: v_readfirstlane_b32 s5, v0
; GFX7LESS_ITERATIVE-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v3, s4
-; GFX7LESS_ITERATIVE-NEXT: v_sub_i32_e32 v0, vcc, s5, v1
-; GFX7LESS_ITERATIVE-NEXT: v_subb_u32_e32 v1, vcc, v3, v2, vcc
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, s4
+; GFX7LESS_ITERATIVE-NEXT: v_sub_i32_e32 v0, vcc, s5, v4
+; GFX7LESS_ITERATIVE-NEXT: v_subb_u32_e32 v1, vcc, v1, v5, vcc
; GFX7LESS_ITERATIVE-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX7LESS_ITERATIVE-NEXT: s_endpgm
;
; GFX8_ITERATIVE-LABEL: sub_i64_varying:
; GFX8_ITERATIVE: ; %bb.0: ; %entry
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
-; GFX8_ITERATIVE-NEXT: s_mov_b64 s[6:7], 0
-; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
+; GFX8_ITERATIVE-NEXT: s_mov_b64 s[8:9], 0
+; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX8_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s2
-; GFX8_ITERATIVE-NEXT: v_readlane_b32 s8, v0, s2
-; GFX8_ITERATIVE-NEXT: v_readlane_b32 s3, v3, s2
-; GFX8_ITERATIVE-NEXT: v_writelane_b32 v1, s6, m0
-; GFX8_ITERATIVE-NEXT: s_add_u32 s6, s6, s8
-; GFX8_ITERATIVE-NEXT: v_writelane_b32 v2, s7, m0
-; GFX8_ITERATIVE-NEXT: s_addc_u32 s7, s7, s3
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s2
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s3, v1, s2
+; GFX8_ITERATIVE-NEXT: v_writelane_b32 v4, s8, m0
+; GFX8_ITERATIVE-NEXT: s_add_u32 s8, s8, s6
+; GFX8_ITERATIVE-NEXT: v_writelane_b32 v5, s9, m0
+; GFX8_ITERATIVE-NEXT: s_addc_u32 s9, s9, s3
; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
; GFX8_ITERATIVE-NEXT: s_cmp_lg_u64 s[0:1], 0
@@ -5936,50 +6686,70 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX8_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX8_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX8_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr3_vgpr4
+; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX8_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX8_ITERATIVE-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GFX8_ITERATIVE-NEXT: s_cbranch_execz .LBB11_4
+; GFX8_ITERATIVE-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
+; GFX8_ITERATIVE-NEXT: s_cbranch_execz .LBB11_6
; GFX8_ITERATIVE-NEXT: ; %bb.3:
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v3, s6
-; GFX8_ITERATIVE-NEXT: s_mov_b32 s11, 0xf000
-; GFX8_ITERATIVE-NEXT: s_mov_b32 s10, -1
; GFX8_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8_ITERATIVE-NEXT: s_mov_b32 s8, s2
-; GFX8_ITERATIVE-NEXT: s_mov_b32 s9, s3
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, s7
-; GFX8_ITERATIVE-NEXT: buffer_atomic_sub_x2 v[3:4], off, s[8:11], 0 glc
+; GFX8_ITERATIVE-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
+; GFX8_ITERATIVE-NEXT: s_mov_b64 s[12:13], 0
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v6, s9
+; GFX8_ITERATIVE-NEXT: s_mov_b32 s7, 0xf000
+; GFX8_ITERATIVE-NEXT: s_mov_b32 s6, -1
+; GFX8_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, s4
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, s5
+; GFX8_ITERATIVE-NEXT: s_mov_b32 s4, s2
+; GFX8_ITERATIVE-NEXT: s_mov_b32 s5, s3
+; GFX8_ITERATIVE-NEXT: .LBB11_4: ; %atomicrmw.start
+; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v10, v1
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v9, v0
+; GFX8_ITERATIVE-NEXT: v_subrev_u32_e32 v7, vcc, s8, v9
+; GFX8_ITERATIVE-NEXT: v_subb_u32_e32 v8, vcc, v10, v6, vcc
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, v7
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, v8
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v2, v9
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v3, v10
+; GFX8_ITERATIVE-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX8_ITERATIVE-NEXT: s_waitcnt vmcnt(0)
; GFX8_ITERATIVE-NEXT: buffer_wbinvl1_vol
-; GFX8_ITERATIVE-NEXT: .LBB11_4:
-; GFX8_ITERATIVE-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX8_ITERATIVE-NEXT: v_readfirstlane_b32 s4, v4
-; GFX8_ITERATIVE-NEXT: v_readfirstlane_b32 s5, v3
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v3, s4
-; GFX8_ITERATIVE-NEXT: v_sub_u32_e32 v0, vcc, s5, v1
+; GFX8_ITERATIVE-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX8_ITERATIVE-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
+; GFX8_ITERATIVE-NEXT: s_andn2_b64 exec, exec, s[12:13]
+; GFX8_ITERATIVE-NEXT: s_cbranch_execnz .LBB11_4
+; GFX8_ITERATIVE-NEXT: ; %bb.5: ; %Flow
+; GFX8_ITERATIVE-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX8_ITERATIVE-NEXT: .LBB11_6: ; %Flow4
+; GFX8_ITERATIVE-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX8_ITERATIVE-NEXT: v_readfirstlane_b32 s4, v1
+; GFX8_ITERATIVE-NEXT: v_readfirstlane_b32 s5, v0
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, s4
+; GFX8_ITERATIVE-NEXT: v_sub_u32_e32 v0, vcc, s5, v4
; GFX8_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX8_ITERATIVE-NEXT: s_mov_b32 s3, 0xf000
; GFX8_ITERATIVE-NEXT: s_mov_b32 s2, -1
-; GFX8_ITERATIVE-NEXT: v_subb_u32_e32 v1, vcc, v3, v2, vcc
+; GFX8_ITERATIVE-NEXT: v_subb_u32_e32 v1, vcc, v1, v5, vcc
; GFX8_ITERATIVE-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX8_ITERATIVE-NEXT: s_endpgm
;
; GFX9_ITERATIVE-LABEL: sub_i64_varying:
; GFX9_ITERATIVE: ; %bb.0: ; %entry
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
-; GFX9_ITERATIVE-NEXT: s_mov_b64 s[6:7], 0
-; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
+; GFX9_ITERATIVE-NEXT: s_mov_b64 s[8:9], 0
+; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX9_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s2
-; GFX9_ITERATIVE-NEXT: v_readlane_b32 s8, v0, s2
-; GFX9_ITERATIVE-NEXT: v_readlane_b32 s3, v3, s2
-; GFX9_ITERATIVE-NEXT: v_writelane_b32 v1, s6, m0
-; GFX9_ITERATIVE-NEXT: s_add_u32 s6, s6, s8
-; GFX9_ITERATIVE-NEXT: v_writelane_b32 v2, s7, m0
-; GFX9_ITERATIVE-NEXT: s_addc_u32 s7, s7, s3
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s2
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s3, v1, s2
+; GFX9_ITERATIVE-NEXT: v_writelane_b32 v4, s8, m0
+; GFX9_ITERATIVE-NEXT: s_add_u32 s8, s8, s6
+; GFX9_ITERATIVE-NEXT: v_writelane_b32 v5, s9, m0
+; GFX9_ITERATIVE-NEXT: s_addc_u32 s9, s9, s3
; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
; GFX9_ITERATIVE-NEXT: s_cmp_lg_u64 s[0:1], 0
@@ -5989,49 +6759,69 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX9_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX9_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr3_vgpr4
+; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX9_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX9_ITERATIVE-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GFX9_ITERATIVE-NEXT: s_cbranch_execz .LBB11_4
+; GFX9_ITERATIVE-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
+; GFX9_ITERATIVE-NEXT: s_cbranch_execz .LBB11_6
; GFX9_ITERATIVE-NEXT: ; %bb.3:
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v3, s6
-; GFX9_ITERATIVE-NEXT: s_mov_b32 s11, 0xf000
-; GFX9_ITERATIVE-NEXT: s_mov_b32 s10, -1
; GFX9_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9_ITERATIVE-NEXT: s_mov_b32 s8, s2
-; GFX9_ITERATIVE-NEXT: s_mov_b32 s9, s3
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, s7
-; GFX9_ITERATIVE-NEXT: buffer_atomic_sub_x2 v[3:4], off, s[8:11], 0 glc
+; GFX9_ITERATIVE-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
+; GFX9_ITERATIVE-NEXT: s_mov_b64 s[12:13], 0
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v6, s9
+; GFX9_ITERATIVE-NEXT: s_mov_b32 s7, 0xf000
+; GFX9_ITERATIVE-NEXT: s_mov_b32 s6, -1
+; GFX9_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, s4
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, s5
+; GFX9_ITERATIVE-NEXT: s_mov_b32 s4, s2
+; GFX9_ITERATIVE-NEXT: s_mov_b32 s5, s3
+; GFX9_ITERATIVE-NEXT: .LBB11_4: ; %atomicrmw.start
+; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v10, v1
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v9, v0
+; GFX9_ITERATIVE-NEXT: v_subrev_co_u32_e32 v7, vcc, s8, v9
+; GFX9_ITERATIVE-NEXT: v_subb_co_u32_e32 v8, vcc, v10, v6, vcc
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, v7
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, v8
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v2, v9
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v3, v10
+; GFX9_ITERATIVE-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX9_ITERATIVE-NEXT: s_waitcnt vmcnt(0)
; GFX9_ITERATIVE-NEXT: buffer_wbinvl1_vol
-; GFX9_ITERATIVE-NEXT: .LBB11_4:
-; GFX9_ITERATIVE-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9_ITERATIVE-NEXT: v_readfirstlane_b32 s4, v4
-; GFX9_ITERATIVE-NEXT: v_readfirstlane_b32 s5, v3
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v3, s4
-; GFX9_ITERATIVE-NEXT: v_sub_co_u32_e32 v0, vcc, s5, v1
+; GFX9_ITERATIVE-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
+; GFX9_ITERATIVE-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
+; GFX9_ITERATIVE-NEXT: s_andn2_b64 exec, exec, s[12:13]
+; GFX9_ITERATIVE-NEXT: s_cbranch_execnz .LBB11_4
+; GFX9_ITERATIVE-NEXT: ; %bb.5: ; %Flow
+; GFX9_ITERATIVE-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX9_ITERATIVE-NEXT: .LBB11_6: ; %Flow4
+; GFX9_ITERATIVE-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX9_ITERATIVE-NEXT: v_readfirstlane_b32 s4, v1
+; GFX9_ITERATIVE-NEXT: v_readfirstlane_b32 s5, v0
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, s4
+; GFX9_ITERATIVE-NEXT: v_sub_co_u32_e32 v0, vcc, s5, v4
; GFX9_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
; GFX9_ITERATIVE-NEXT: s_mov_b32 s3, 0xf000
; GFX9_ITERATIVE-NEXT: s_mov_b32 s2, -1
-; GFX9_ITERATIVE-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v2, vcc
+; GFX9_ITERATIVE-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v5, vcc
; GFX9_ITERATIVE-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX9_ITERATIVE-NEXT: s_endpgm
;
; GFX1064_ITERATIVE-LABEL: sub_i64_varying:
; GFX1064_ITERATIVE: ; %bb.0: ; %entry
-; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[6:7], 0
-; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
+; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[8:9], 0
+; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1064_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s2
-; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s8, v3, s2
-; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v1, s6, s2
-; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v2, s7, s2
-; GFX1064_ITERATIVE-NEXT: s_add_u32 s6, s6, s3
-; GFX1064_ITERATIVE-NEXT: s_addc_u32 s7, s7, s8
+; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s6, v1, s2
+; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v4, s8, s2
+; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v5, s9, s2
+; GFX1064_ITERATIVE-NEXT: s_add_u32 s8, s8, s3
+; GFX1064_ITERATIVE-NEXT: s_addc_u32 s9, s9, s6
; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
; GFX1064_ITERATIVE-NEXT: s_cmp_lg_u64 s[0:1], 0
@@ -6039,32 +6829,50 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1064_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1064_ITERATIVE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr3_vgpr4
; GFX1064_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1064_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1064_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX1064_ITERATIVE-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GFX1064_ITERATIVE-NEXT: s_cbranch_execz .LBB11_4
+; GFX1064_ITERATIVE-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
+; GFX1064_ITERATIVE-NEXT: s_cbranch_execz .LBB11_6
; GFX1064_ITERATIVE-NEXT: ; %bb.3:
-; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v3, s6
-; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v4, s7
-; GFX1064_ITERATIVE-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1064_ITERATIVE-NEXT: s_mov_b32 s10, -1
; GFX1064_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064_ITERATIVE-NEXT: s_mov_b32 s8, s2
-; GFX1064_ITERATIVE-NEXT: s_mov_b32 s9, s3
-; GFX1064_ITERATIVE-NEXT: buffer_atomic_sub_x2 v[3:4], off, s[8:11], 0 glc
+; GFX1064_ITERATIVE-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
+; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[12:13], 0
+; GFX1064_ITERATIVE-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064_ITERATIVE-NEXT: s_mov_b32 s6, -1
+; GFX1064_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v0, s4
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v1, s5
+; GFX1064_ITERATIVE-NEXT: s_mov_b32 s4, s2
+; GFX1064_ITERATIVE-NEXT: s_mov_b32 s5, s3
+; GFX1064_ITERATIVE-NEXT: .LBB11_4: ; %atomicrmw.start
+; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v9, v1
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v8, v0
+; GFX1064_ITERATIVE-NEXT: v_sub_co_u32 v6, vcc, v8, s8
+; GFX1064_ITERATIVE-NEXT: v_subrev_co_ci_u32_e32 v7, vcc, s9, v9, vcc
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v0, v6
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v2, v8
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v1, v7
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v3, v9
+; GFX1064_ITERATIVE-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX1064_ITERATIVE-NEXT: s_waitcnt vmcnt(0)
; GFX1064_ITERATIVE-NEXT: buffer_gl1_inv
; GFX1064_ITERATIVE-NEXT: buffer_gl0_inv
-; GFX1064_ITERATIVE-NEXT: .LBB11_4:
-; GFX1064_ITERATIVE-NEXT: s_waitcnt_depctr 0xffe3
-; GFX1064_ITERATIVE-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX1064_ITERATIVE-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX1064_ITERATIVE-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
+; GFX1064_ITERATIVE-NEXT: s_andn2_b64 exec, exec, s[12:13]
+; GFX1064_ITERATIVE-NEXT: s_cbranch_execnz .LBB11_4
+; GFX1064_ITERATIVE-NEXT: ; %bb.5: ; %Flow
+; GFX1064_ITERATIVE-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1064_ITERATIVE-NEXT: .LBB11_6: ; %Flow4
+; GFX1064_ITERATIVE-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX1064_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v3
-; GFX1064_ITERATIVE-NEXT: v_readfirstlane_b32 s3, v4
-; GFX1064_ITERATIVE-NEXT: v_sub_co_u32 v0, vcc, s2, v1
-; GFX1064_ITERATIVE-NEXT: v_sub_co_ci_u32_e32 v1, vcc, s3, v2, vcc
+; GFX1064_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1064_ITERATIVE-NEXT: v_readfirstlane_b32 s3, v1
+; GFX1064_ITERATIVE-NEXT: v_sub_co_u32 v0, vcc, s2, v4
+; GFX1064_ITERATIVE-NEXT: v_sub_co_ci_u32_e32 v1, vcc, s3, v5, vcc
; GFX1064_ITERATIVE-NEXT: s_mov_b32 s3, 0x31016000
; GFX1064_ITERATIVE-NEXT: s_mov_b32 s2, -1
; GFX1064_ITERATIVE-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -6072,19 +6880,19 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1032_ITERATIVE-LABEL: sub_i64_varying:
; GFX1032_ITERATIVE: ; %bb.0: ; %entry
-; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX1032_ITERATIVE-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032_ITERATIVE-NEXT: s_mov_b64 s[6:7], 0
-; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
+; GFX1032_ITERATIVE-NEXT: s_mov_b64 s[8:9], 0
+; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1032_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s1, s0
; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s2, v0, s1
-; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s3, v3, s1
-; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s6, s1
-; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s7, s1
-; GFX1032_ITERATIVE-NEXT: s_add_u32 s6, s6, s2
-; GFX1032_ITERATIVE-NEXT: s_addc_u32 s7, s7, s3
+; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s3, v1, s1
+; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v4, s8, s1
+; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v5, s9, s1
+; GFX1032_ITERATIVE-NEXT: s_add_u32 s8, s8, s2
+; GFX1032_ITERATIVE-NEXT: s_addc_u32 s9, s9, s3
; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s1, 1, s1
; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s0, s0, s1
; GFX1032_ITERATIVE-NEXT: s_cmp_lg_u32 s0, 0
@@ -6092,31 +6900,49 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1032_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1032_ITERATIVE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr3_vgpr4
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s11, 0
; GFX1032_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1032_ITERATIVE-NEXT: s_and_saveexec_b32 s4, vcc_lo
-; GFX1032_ITERATIVE-NEXT: s_xor_b32 s4, exec_lo, s4
-; GFX1032_ITERATIVE-NEXT: s_cbranch_execz .LBB11_4
+; GFX1032_ITERATIVE-NEXT: s_xor_b32 s10, exec_lo, s4
+; GFX1032_ITERATIVE-NEXT: s_cbranch_execz .LBB11_6
; GFX1032_ITERATIVE-NEXT: ; %bb.3:
-; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v3, s6
-; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v4, s7
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s10, -1
; GFX1032_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s8, s2
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s9, s3
-; GFX1032_ITERATIVE-NEXT: buffer_atomic_sub_x2 v[3:4], off, s[8:11], 0 glc
+; GFX1032_ITERATIVE-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s6, -1
+; GFX1032_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v0, s4
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v1, s5
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s4, s2
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s5, s3
+; GFX1032_ITERATIVE-NEXT: .LBB11_4: ; %atomicrmw.start
+; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v9, v1
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v8, v0
+; GFX1032_ITERATIVE-NEXT: v_sub_co_u32 v6, vcc_lo, v8, s8
+; GFX1032_ITERATIVE-NEXT: v_subrev_co_ci_u32_e32 v7, vcc_lo, s9, v9, vcc_lo
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v0, v6
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v2, v8
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v1, v7
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v3, v9
+; GFX1032_ITERATIVE-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX1032_ITERATIVE-NEXT: s_waitcnt vmcnt(0)
; GFX1032_ITERATIVE-NEXT: buffer_gl1_inv
; GFX1032_ITERATIVE-NEXT: buffer_gl0_inv
-; GFX1032_ITERATIVE-NEXT: .LBB11_4:
-; GFX1032_ITERATIVE-NEXT: s_waitcnt_depctr 0xffe3
-; GFX1032_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX1032_ITERATIVE-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
+; GFX1032_ITERATIVE-NEXT: s_or_b32 s11, vcc_lo, s11
+; GFX1032_ITERATIVE-NEXT: s_andn2_b32 exec_lo, exec_lo, s11
+; GFX1032_ITERATIVE-NEXT: s_cbranch_execnz .LBB11_4
+; GFX1032_ITERATIVE-NEXT: ; %bb.5: ; %Flow
+; GFX1032_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s11
+; GFX1032_ITERATIVE-NEXT: .LBB11_6: ; %Flow4
+; GFX1032_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s10
; GFX1032_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v3
-; GFX1032_ITERATIVE-NEXT: v_readfirstlane_b32 s3, v4
-; GFX1032_ITERATIVE-NEXT: v_sub_co_u32 v0, vcc_lo, s2, v1
-; GFX1032_ITERATIVE-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, s3, v2, vcc_lo
+; GFX1032_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1032_ITERATIVE-NEXT: v_readfirstlane_b32 s3, v1
+; GFX1032_ITERATIVE-NEXT: v_sub_co_u32 v0, vcc_lo, s2, v4
+; GFX1032_ITERATIVE-NEXT: v_sub_co_ci_u32_e32 v1, vcc_lo, s3, v5, vcc_lo
; GFX1032_ITERATIVE-NEXT: s_mov_b32 s3, 0x31016000
; GFX1032_ITERATIVE-NEXT: s_mov_b32 s2, -1
; GFX1032_ITERATIVE-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -6124,21 +6950,21 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1164_ITERATIVE-LABEL: sub_i64_varying:
; GFX1164_ITERATIVE: ; %bb.0: ; %entry
-; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
-; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
+; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[6:7], 0
-; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[8:9], 0
+; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1164_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s3, v2, s2
-; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s8, v3, s2
-; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v0, s6, s2
-; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v1, s7, s2
-; GFX1164_ITERATIVE-NEXT: s_add_u32 s6, s6, s3
-; GFX1164_ITERATIVE-NEXT: s_addc_u32 s7, s7, s8
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v1, s2
+; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v4, s8, s2
+; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v5, s9, s2
+; GFX1164_ITERATIVE-NEXT: s_add_u32 s8, s8, s3
+; GFX1164_ITERATIVE-NEXT: s_addc_u32 s9, s9, s6
; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
@@ -6146,35 +6972,59 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1164_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_1
; GFX1164_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1164_ITERATIVE-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1164_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v2, exec_hi, v2
-; GFX1164_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
-; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX1164_ITERATIVE-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
+; GFX1164_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
+; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1164_ITERATIVE-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164_ITERATIVE-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GFX1164_ITERATIVE-NEXT: s_cbranch_execz .LBB11_4
+; GFX1164_ITERATIVE-NEXT: s_xor_b64 s[10:11], exec, s[4:5]
+; GFX1164_ITERATIVE-NEXT: s_cbranch_execz .LBB11_6
; GFX1164_ITERATIVE-NEXT: ; %bb.3:
-; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v2, s6
-; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v3, s7
-; GFX1164_ITERATIVE-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1164_ITERATIVE-NEXT: s_mov_b32 s10, -1
; GFX1164_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164_ITERATIVE-NEXT: s_mov_b32 s8, s2
-; GFX1164_ITERATIVE-NEXT: s_mov_b32 s9, s3
-; GFX1164_ITERATIVE-NEXT: buffer_atomic_sub_u64 v[2:3], off, s[8:11], 0 glc
+; GFX1164_ITERATIVE-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
+; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[12:13], 0
+; GFX1164_ITERATIVE-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164_ITERATIVE-NEXT: s_mov_b32 s6, -1
+; GFX1164_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v0, s4
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v1, s5
+; GFX1164_ITERATIVE-NEXT: s_mov_b32 s4, s2
+; GFX1164_ITERATIVE-NEXT: s_mov_b32 s5, s3
+; GFX1164_ITERATIVE-NEXT: .LBB11_4: ; %atomicrmw.start
+; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v9, v1
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v8, v0
+; GFX1164_ITERATIVE-NEXT: v_sub_co_u32 v6, vcc, v8, s8
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_ITERATIVE-NEXT: v_subrev_co_ci_u32_e64 v7, null, s9, v9, vcc
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v0, v6
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v2, v8
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v1, v7
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v3, v9
+; GFX1164_ITERATIVE-NEXT: buffer_atomic_cmpswap_b64 v[0:3], off, s[4:7], 0 glc
; GFX1164_ITERATIVE-NEXT: s_waitcnt vmcnt(0)
; GFX1164_ITERATIVE-NEXT: buffer_gl1_inv
; GFX1164_ITERATIVE-NEXT: buffer_gl0_inv
-; GFX1164_ITERATIVE-NEXT: .LBB11_4:
-; GFX1164_ITERATIVE-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX1164_ITERATIVE-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX1164_ITERATIVE-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 exec, exec, s[12:13]
+; GFX1164_ITERATIVE-NEXT: s_cbranch_execnz .LBB11_4
+; GFX1164_ITERATIVE-NEXT: ; %bb.5: ; %Flow
+; GFX1164_ITERATIVE-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1164_ITERATIVE-NEXT: .LBB11_6: ; %Flow4
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1164_ITERATIVE-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX1164_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v2
-; GFX1164_ITERATIVE-NEXT: v_readfirstlane_b32 s3, v3
-; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164_ITERATIVE-NEXT: v_sub_co_u32 v0, vcc, s2, v0
-; GFX1164_ITERATIVE-NEXT: v_sub_co_ci_u32_e64 v1, null, s3, v1, vcc
+; GFX1164_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1164_ITERATIVE-NEXT: v_readfirstlane_b32 s3, v1
+; GFX1164_ITERATIVE-NEXT: v_sub_co_u32 v0, vcc, s2, v4
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1164_ITERATIVE-NEXT: v_sub_co_ci_u32_e64 v1, null, s3, v5, vcc
; GFX1164_ITERATIVE-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164_ITERATIVE-NEXT: s_mov_b32 s2, -1
; GFX1164_ITERATIVE-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 0
@@ -6182,20 +7032,20 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1132_ITERATIVE-LABEL: sub_i64_varying:
; GFX1132_ITERATIVE: ; %bb.0: ; %entry
-; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v2, 0x3ff, v0
+; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX1132_ITERATIVE-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132_ITERATIVE-NEXT: s_mov_b64 s[6:7], 0
-; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1132_ITERATIVE-NEXT: s_mov_b64 s[8:9], 0
+; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX1132_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s1, s0
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s2, v2, s1
-; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s3, v3, s1
-; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s6, s1
-; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v1, s7, s1
-; GFX1132_ITERATIVE-NEXT: s_add_u32 s6, s6, s2
-; GFX1132_ITERATIVE-NEXT: s_addc_u32 s7, s7, s3
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s2, v0, s1
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s3, v1, s1
+; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v4, s8, s1
+; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v5, s9, s1
+; GFX1132_ITERATIVE-NEXT: s_add_u32 s8, s8, s2
+; GFX1132_ITERATIVE-NEXT: s_addc_u32 s9, s9, s3
; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s1, 1, s1
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s0, s0, s1
@@ -6203,32 +7053,54 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1132_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_1
; GFX1132_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1132_ITERATIVE-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
+; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s11, 0
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1132_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
-; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX1132_ITERATIVE-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1132_ITERATIVE-NEXT: s_and_saveexec_b32 s4, vcc_lo
-; GFX1132_ITERATIVE-NEXT: s_xor_b32 s4, exec_lo, s4
-; GFX1132_ITERATIVE-NEXT: s_cbranch_execz .LBB11_4
+; GFX1132_ITERATIVE-NEXT: s_xor_b32 s10, exec_lo, s4
+; GFX1132_ITERATIVE-NEXT: s_cbranch_execz .LBB11_6
; GFX1132_ITERATIVE-NEXT: ; %bb.3:
-; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s7
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s10, -1
; GFX1132_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s8, s2
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s9, s3
-; GFX1132_ITERATIVE-NEXT: buffer_atomic_sub_u64 v[2:3], off, s[8:11], 0 glc
+; GFX1132_ITERATIVE-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s6, -1
+; GFX1132_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s4, s2
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s5, s3
+; GFX1132_ITERATIVE-NEXT: .LBB11_4: ; %atomicrmw.start
+; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
+; GFX1132_ITERATIVE-NEXT: v_sub_co_u32 v6, vcc_lo, v8, s8
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132_ITERATIVE-NEXT: v_subrev_co_ci_u32_e64 v7, null, s9, v9, vcc_lo
+; GFX1132_ITERATIVE-NEXT: v_mov_b32_e32 v0, v6
+; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1132_ITERATIVE-NEXT: v_mov_b32_e32 v1, v7
+; GFX1132_ITERATIVE-NEXT: buffer_atomic_cmpswap_b64 v[0:3], off, s[4:7], 0 glc
; GFX1132_ITERATIVE-NEXT: s_waitcnt vmcnt(0)
; GFX1132_ITERATIVE-NEXT: buffer_gl1_inv
; GFX1132_ITERATIVE-NEXT: buffer_gl0_inv
-; GFX1132_ITERATIVE-NEXT: .LBB11_4:
-; GFX1132_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX1132_ITERATIVE-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
+; GFX1132_ITERATIVE-NEXT: s_or_b32 s11, vcc_lo, s11
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 exec_lo, exec_lo, s11
+; GFX1132_ITERATIVE-NEXT: s_cbranch_execnz .LBB11_4
+; GFX1132_ITERATIVE-NEXT: ; %bb.5: ; %Flow
+; GFX1132_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s11
+; GFX1132_ITERATIVE-NEXT: .LBB11_6: ; %Flow4
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1132_ITERATIVE-NEXT: s_or_b32 exec_lo, exec_lo, s10
; GFX1132_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v2
-; GFX1132_ITERATIVE-NEXT: v_readfirstlane_b32 s3, v3
-; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132_ITERATIVE-NEXT: v_sub_co_u32 v0, vcc_lo, s2, v0
-; GFX1132_ITERATIVE-NEXT: v_sub_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
+; GFX1132_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132_ITERATIVE-NEXT: v_readfirstlane_b32 s3, v1
+; GFX1132_ITERATIVE-NEXT: v_sub_co_u32 v0, vcc_lo, s2, v4
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132_ITERATIVE-NEXT: v_sub_co_ci_u32_e64 v1, null, s3, v5, vcc_lo
; GFX1132_ITERATIVE-NEXT: s_mov_b32 s3, 0x31016000
; GFX1132_ITERATIVE-NEXT: s_mov_b32 s2, -1
; GFX1132_ITERATIVE-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 0
@@ -6344,20 +7216,39 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS_DPP-LABEL: sub_i64_varying:
; GFX7LESS_DPP: ; %bb.0: ; %entry
; GFX7LESS_DPP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7LESS_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS_DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
+; GFX7LESS_DPP-NEXT: s_mov_b64 s[8:9], 0
; GFX7LESS_DPP-NEXT: s_mov_b32 s7, 0xf000
-; GFX7LESS_DPP-NEXT: s_mov_b32 s6, -1
-; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v1, 0
-; GFX7LESS_DPP-NEXT: s_mov_b32 s10, s6
-; GFX7LESS_DPP-NEXT: s_mov_b32 s11, s7
; GFX7LESS_DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS_DPP-NEXT: s_mov_b32 s8, s2
-; GFX7LESS_DPP-NEXT: s_mov_b32 s9, s3
-; GFX7LESS_DPP-NEXT: buffer_atomic_sub_x2 v[0:1], off, s[8:11], 0 glc
+; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v3, s4
+; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v4, s5
+; GFX7LESS_DPP-NEXT: s_mov_b32 s6, -1
+; GFX7LESS_DPP-NEXT: s_mov_b32 s4, s2
+; GFX7LESS_DPP-NEXT: s_mov_b32 s5, s3
+; GFX7LESS_DPP-NEXT: .LBB11_1: ; %atomicrmw.start
+; GFX7LESS_DPP-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS_DPP-NEXT: v_sub_i32_e32 v1, vcc, v3, v0
+; GFX7LESS_DPP-NEXT: v_subbrev_u32_e32 v2, vcc, 0, v4, vcc
+; GFX7LESS_DPP-NEXT: s_waitcnt expcnt(0)
+; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v8, v4
+; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v7, v3
+; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v6, v2
+; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v5, v1
+; GFX7LESS_DPP-NEXT: buffer_atomic_cmpswap_x2 v[5:8], off, s[4:7], 0 glc
; GFX7LESS_DPP-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS_DPP-NEXT: buffer_wbinvl1
-; GFX7LESS_DPP-NEXT: s_mov_b32 s4, s0
-; GFX7LESS_DPP-NEXT: s_mov_b32 s5, s1
-; GFX7LESS_DPP-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX7LESS_DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[5:6], v[3:4]
+; GFX7LESS_DPP-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v3, v5
+; GFX7LESS_DPP-NEXT: v_mov_b32_e32 v4, v6
+; GFX7LESS_DPP-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX7LESS_DPP-NEXT: s_cbranch_execnz .LBB11_1
+; GFX7LESS_DPP-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX7LESS_DPP-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX7LESS_DPP-NEXT: s_mov_b32 s3, 0xf000
+; GFX7LESS_DPP-NEXT: s_mov_b32 s2, -1
+; GFX7LESS_DPP-NEXT: buffer_store_dwordx2 v[5:6], off, s[0:3], 0
; GFX7LESS_DPP-NEXT: s_endpgm
;
; GFX8_DPP-LABEL: sub_i64_varying:
@@ -6408,28 +7299,48 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX8_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
; GFX8_DPP-NEXT: v_addc_u32_e32 v4, vcc, v2, v4, vcc
; GFX8_DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX8_DPP-NEXT: v_readlane_b32 s7, v4, 63
-; GFX8_DPP-NEXT: v_readlane_b32 s6, v3, 63
+; GFX8_DPP-NEXT: v_readlane_b32 s11, v4, 63
+; GFX8_DPP-NEXT: v_readlane_b32 s10, v3, 63
; GFX8_DPP-NEXT: v_mov_b32_dpp v2, v4 wave_shr:1 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: v_mov_b32_dpp v1, v3 wave_shr:1 row_mask:0xf bank_mask:0xf
; GFX8_DPP-NEXT: s_mov_b64 exec, s[4:5]
; GFX8_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v6
; GFX8_DPP-NEXT: ; implicit-def: $vgpr6_vgpr7
-; GFX8_DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX8_DPP-NEXT: s_cbranch_execz .LBB11_2
+; GFX8_DPP-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX8_DPP-NEXT: s_cbranch_execz .LBB11_4
; GFX8_DPP-NEXT: ; %bb.1:
-; GFX8_DPP-NEXT: v_mov_b32_e32 v6, s6
-; GFX8_DPP-NEXT: s_mov_b32 s11, 0xf000
-; GFX8_DPP-NEXT: s_mov_b32 s10, -1
; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8_DPP-NEXT: s_mov_b32 s8, s2
-; GFX8_DPP-NEXT: s_mov_b32 s9, s3
-; GFX8_DPP-NEXT: v_mov_b32_e32 v7, s7
-; GFX8_DPP-NEXT: buffer_atomic_sub_x2 v[6:7], off, s[8:11], 0 glc
+; GFX8_DPP-NEXT: s_load_dwordx2 s[14:15], s[2:3], 0x0
+; GFX8_DPP-NEXT: s_mov_b64 s[12:13], 0
+; GFX8_DPP-NEXT: s_mov_b32 s7, 0xf000
+; GFX8_DPP-NEXT: s_mov_b32 s6, -1
+; GFX8_DPP-NEXT: s_mov_b32 s4, s2
+; GFX8_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8_DPP-NEXT: v_mov_b32_e32 v6, s14
+; GFX8_DPP-NEXT: v_mov_b32_e32 v7, s15
+; GFX8_DPP-NEXT: s_mov_b32 s5, s3
+; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s11
+; GFX8_DPP-NEXT: .LBB11_2: ; %atomicrmw.start
+; GFX8_DPP-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8_DPP-NEXT: v_mov_b32_e32 v11, v7
+; GFX8_DPP-NEXT: v_mov_b32_e32 v10, v6
+; GFX8_DPP-NEXT: v_subrev_u32_e32 v8, vcc, s10, v10
+; GFX8_DPP-NEXT: v_subb_u32_e32 v9, vcc, v11, v0, vcc
+; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v8
+; GFX8_DPP-NEXT: v_mov_b32_e32 v7, v9
+; GFX8_DPP-NEXT: v_mov_b32_e32 v8, v10
+; GFX8_DPP-NEXT: v_mov_b32_e32 v9, v11
+; GFX8_DPP-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 glc
; GFX8_DPP-NEXT: s_waitcnt vmcnt(0)
; GFX8_DPP-NEXT: buffer_wbinvl1_vol
-; GFX8_DPP-NEXT: .LBB11_2:
-; GFX8_DPP-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8_DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[10:11]
+; GFX8_DPP-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
+; GFX8_DPP-NEXT: s_andn2_b64 exec, exec, s[12:13]
+; GFX8_DPP-NEXT: s_cbranch_execnz .LBB11_2
+; GFX8_DPP-NEXT: ; %bb.3: ; %Flow
+; GFX8_DPP-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX8_DPP-NEXT: .LBB11_4: ; %Flow3
+; GFX8_DPP-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8_DPP-NEXT: v_readfirstlane_b32 s4, v7
; GFX8_DPP-NEXT: v_readfirstlane_b32 s5, v6
; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v1
@@ -6491,28 +7402,48 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX9_DPP-NEXT: v_mov_b32_dpp v4, v2 row_bcast:31 row_mask:0xc bank_mask:0xf
; GFX9_DPP-NEXT: v_addc_co_u32_e32 v4, vcc, v2, v4, vcc
; GFX9_DPP-NEXT: v_mov_b32_e32 v2, 0
-; GFX9_DPP-NEXT: v_readlane_b32 s7, v4, 63
-; GFX9_DPP-NEXT: v_readlane_b32 s6, v3, 63
+; GFX9_DPP-NEXT: v_readlane_b32 s11, v4, 63
+; GFX9_DPP-NEXT: v_readlane_b32 s10, v3, 63
; GFX9_DPP-NEXT: v_mov_b32_dpp v2, v4 wave_shr:1 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: v_mov_b32_dpp v1, v3 wave_shr:1 row_mask:0xf bank_mask:0xf
; GFX9_DPP-NEXT: s_mov_b64 exec, s[4:5]
; GFX9_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v6
; GFX9_DPP-NEXT: ; implicit-def: $vgpr6_vgpr7
-; GFX9_DPP-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX9_DPP-NEXT: s_cbranch_execz .LBB11_2
+; GFX9_DPP-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX9_DPP-NEXT: s_cbranch_execz .LBB11_4
; GFX9_DPP-NEXT: ; %bb.1:
-; GFX9_DPP-NEXT: v_mov_b32_e32 v6, s6
-; GFX9_DPP-NEXT: s_mov_b32 s11, 0xf000
-; GFX9_DPP-NEXT: s_mov_b32 s10, -1
; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9_DPP-NEXT: s_mov_b32 s8, s2
-; GFX9_DPP-NEXT: s_mov_b32 s9, s3
-; GFX9_DPP-NEXT: v_mov_b32_e32 v7, s7
-; GFX9_DPP-NEXT: buffer_atomic_sub_x2 v[6:7], off, s[8:11], 0 glc
+; GFX9_DPP-NEXT: s_load_dwordx2 s[14:15], s[2:3], 0x0
+; GFX9_DPP-NEXT: s_mov_b64 s[12:13], 0
+; GFX9_DPP-NEXT: s_mov_b32 s7, 0xf000
+; GFX9_DPP-NEXT: s_mov_b32 s6, -1
+; GFX9_DPP-NEXT: s_mov_b32 s4, s2
+; GFX9_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9_DPP-NEXT: v_mov_b32_e32 v6, s14
+; GFX9_DPP-NEXT: v_mov_b32_e32 v7, s15
+; GFX9_DPP-NEXT: s_mov_b32 s5, s3
+; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s11
+; GFX9_DPP-NEXT: .LBB11_2: ; %atomicrmw.start
+; GFX9_DPP-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9_DPP-NEXT: v_mov_b32_e32 v11, v7
+; GFX9_DPP-NEXT: v_mov_b32_e32 v10, v6
+; GFX9_DPP-NEXT: v_subrev_co_u32_e32 v8, vcc, s10, v10
+; GFX9_DPP-NEXT: v_subb_co_u32_e32 v9, vcc, v11, v0, vcc
+; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v8
+; GFX9_DPP-NEXT: v_mov_b32_e32 v7, v9
+; GFX9_DPP-NEXT: v_mov_b32_e32 v8, v10
+; GFX9_DPP-NEXT: v_mov_b32_e32 v9, v11
+; GFX9_DPP-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 glc
; GFX9_DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9_DPP-NEXT: buffer_wbinvl1_vol
-; GFX9_DPP-NEXT: .LBB11_2:
-; GFX9_DPP-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9_DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[10:11]
+; GFX9_DPP-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
+; GFX9_DPP-NEXT: s_andn2_b64 exec, exec, s[12:13]
+; GFX9_DPP-NEXT: s_cbranch_execnz .LBB11_2
+; GFX9_DPP-NEXT: ; %bb.3: ; %Flow
+; GFX9_DPP-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX9_DPP-NEXT: .LBB11_4: ; %Flow3
+; GFX9_DPP-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX9_DPP-NEXT: v_readfirstlane_b32 s4, v7
; GFX9_DPP-NEXT: v_readfirstlane_b32 s5, v6
; GFX9_DPP-NEXT: v_mov_b32_e32 v6, v1
@@ -6583,49 +7514,66 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1064_DPP-NEXT: v_mov_b32_dpp v7, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1064_DPP-NEXT: v_readlane_b32 s6, v2, 15
; GFX1064_DPP-NEXT: v_readlane_b32 s7, v1, 15
-; GFX1064_DPP-NEXT: v_readlane_b32 s8, v2, 31
-; GFX1064_DPP-NEXT: v_readlane_b32 s9, v1, 31
-; GFX1064_DPP-NEXT: v_readlane_b32 s10, v1, 47
+; GFX1064_DPP-NEXT: v_readlane_b32 s10, v2, 31
+; GFX1064_DPP-NEXT: v_readlane_b32 s11, v1, 31
+; GFX1064_DPP-NEXT: v_readlane_b32 s9, v2, 63
; GFX1064_DPP-NEXT: v_writelane_b32 v7, s6, 16
; GFX1064_DPP-NEXT: v_writelane_b32 v6, s7, 16
-; GFX1064_DPP-NEXT: v_readlane_b32 s6, v1, 63
-; GFX1064_DPP-NEXT: v_readlane_b32 s11, v2, 47
-; GFX1064_DPP-NEXT: v_readlane_b32 s7, v2, 63
-; GFX1064_DPP-NEXT: v_writelane_b32 v7, s8, 32
-; GFX1064_DPP-NEXT: v_writelane_b32 v6, s9, 32
+; GFX1064_DPP-NEXT: v_readlane_b32 s6, v2, 47
+; GFX1064_DPP-NEXT: v_readlane_b32 s8, v1, 63
+; GFX1064_DPP-NEXT: v_readlane_b32 s12, v1, 47
+; GFX1064_DPP-NEXT: v_writelane_b32 v7, s10, 32
+; GFX1064_DPP-NEXT: v_writelane_b32 v6, s11, 32
; GFX1064_DPP-NEXT: s_mov_b64 exec, s[4:5]
; GFX1064_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[8:9], -1
-; GFX1064_DPP-NEXT: s_mov_b64 s[4:5], s[6:7]
-; GFX1064_DPP-NEXT: v_writelane_b32 v7, s11, 48
-; GFX1064_DPP-NEXT: v_writelane_b32 v6, s10, 48
-; GFX1064_DPP-NEXT: s_mov_b64 exec, s[8:9]
+; GFX1064_DPP-NEXT: s_or_saveexec_b64 s[4:5], -1
+; GFX1064_DPP-NEXT: v_writelane_b32 v7, s6, 48
+; GFX1064_DPP-NEXT: v_writelane_b32 v6, s12, 48
+; GFX1064_DPP-NEXT: s_mov_b64 exec, s[4:5]
; GFX1064_DPP-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064_DPP-NEXT: s_mov_b32 s6, -1
; GFX1064_DPP-NEXT: ; implicit-def: $vgpr8_vgpr9
-; GFX1064_DPP-NEXT: s_and_saveexec_b64 s[8:9], vcc
-; GFX1064_DPP-NEXT: s_cbranch_execz .LBB11_2
+; GFX1064_DPP-NEXT: s_and_saveexec_b64 s[10:11], vcc
+; GFX1064_DPP-NEXT: s_cbranch_execz .LBB11_4
; GFX1064_DPP-NEXT: ; %bb.1:
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v9, s5
-; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, s4
+; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064_DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
+; GFX1064_DPP-NEXT: s_mov_b64 s[12:13], 0
; GFX1064_DPP-NEXT: s_mov_b32 s7, 0x31016000
; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v9, s5
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, s4
; GFX1064_DPP-NEXT: s_mov_b32 s4, s2
; GFX1064_DPP-NEXT: s_mov_b32 s5, s3
-; GFX1064_DPP-NEXT: buffer_atomic_sub_x2 v[8:9], off, s[4:7], 0 glc
+; GFX1064_DPP-NEXT: .LBB11_2: ; %atomicrmw.start
+; GFX1064_DPP-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v13, v9
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v12, v8
+; GFX1064_DPP-NEXT: v_sub_co_u32 v10, vcc, v12, s8
+; GFX1064_DPP-NEXT: v_subrev_co_ci_u32_e32 v11, vcc, s9, v13, vcc
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v8, v10
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v9, v11
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v10, v12
+; GFX1064_DPP-NEXT: v_mov_b32_e32 v11, v13
+; GFX1064_DPP-NEXT: buffer_atomic_cmpswap_x2 v[8:11], off, s[4:7], 0 glc
; GFX1064_DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1064_DPP-NEXT: buffer_gl1_inv
; GFX1064_DPP-NEXT: buffer_gl0_inv
-; GFX1064_DPP-NEXT: .LBB11_2:
-; GFX1064_DPP-NEXT: s_waitcnt_depctr 0xffe3
-; GFX1064_DPP-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1064_DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[8:9], v[12:13]
+; GFX1064_DPP-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
+; GFX1064_DPP-NEXT: s_andn2_b64 exec, exec, s[12:13]
+; GFX1064_DPP-NEXT: s_cbranch_execnz .LBB11_2
+; GFX1064_DPP-NEXT: ; %bb.3: ; %Flow
+; GFX1064_DPP-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1064_DPP-NEXT: .LBB11_4: ; %Flow3
+; GFX1064_DPP-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX1064_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064_DPP-NEXT: v_readfirstlane_b32 s2, v8
; GFX1064_DPP-NEXT: v_mov_b32_e32 v10, v6
; GFX1064_DPP-NEXT: v_mov_b32_e32 v11, v7
; GFX1064_DPP-NEXT: v_readfirstlane_b32 s3, v9
; GFX1064_DPP-NEXT: v_sub_co_u32 v8, vcc, s2, v10
-; GFX1064_DPP-NEXT: s_mov_b32 s2, s6
+; GFX1064_DPP-NEXT: s_mov_b32 s2, -1
; GFX1064_DPP-NEXT: v_sub_co_ci_u32_e32 v9, vcc, s3, v11, vcc
; GFX1064_DPP-NEXT: s_mov_b32 s3, 0x31016000
; GFX1064_DPP-NEXT: buffer_store_dwordx2 v[8:9], off, s[0:3], 0
@@ -6670,47 +7618,65 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1032_DPP-NEXT: v_mov_b32_dpp v5, v6 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1032_DPP-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1032_DPP-NEXT: s_or_saveexec_b32 s6, -1
+; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -1
; GFX1032_DPP-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3
; GFX1032_DPP-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo
-; GFX1032_DPP-NEXT: v_readlane_b32 s4, v1, 31
+; GFX1032_DPP-NEXT: v_readlane_b32 s8, v1, 31
; GFX1032_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX1032_DPP-NEXT: v_readlane_b32 s8, v2, 15
-; GFX1032_DPP-NEXT: v_readlane_b32 s5, v2, 31
+; GFX1032_DPP-NEXT: v_readlane_b32 s6, v2, 15
+; GFX1032_DPP-NEXT: v_readlane_b32 s9, v2, 31
; GFX1032_DPP-NEXT: v_mov_b32_dpp v8, v2 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX1032_DPP-NEXT: v_readlane_b32 s7, v1, 15
-; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s6
+; GFX1032_DPP-NEXT: v_readlane_b32 s5, v1, 15
+; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4
; GFX1032_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1032_DPP-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1032_DPP-NEXT: v_writelane_b32 v8, s8, 16
-; GFX1032_DPP-NEXT: v_writelane_b32 v7, s7, 16
-; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s6
+; GFX1032_DPP-NEXT: s_or_saveexec_b32 s4, -1
+; GFX1032_DPP-NEXT: v_writelane_b32 v8, s6, 16
+; GFX1032_DPP-NEXT: v_writelane_b32 v7, s5, 16
+; GFX1032_DPP-NEXT: s_mov_b32 exec_lo, s4
; GFX1032_DPP-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032_DPP-NEXT: s_mov_b32 s6, -1
; GFX1032_DPP-NEXT: ; implicit-def: $vgpr9_vgpr10
-; GFX1032_DPP-NEXT: s_and_saveexec_b32 s8, vcc_lo
-; GFX1032_DPP-NEXT: s_cbranch_execz .LBB11_2
+; GFX1032_DPP-NEXT: s_and_saveexec_b32 s10, vcc_lo
+; GFX1032_DPP-NEXT: s_cbranch_execz .LBB11_4
; GFX1032_DPP-NEXT: ; %bb.1:
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v10, s5
-; GFX1032_DPP-NEXT: v_mov_b32_e32 v9, s4
+; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032_DPP-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
+; GFX1032_DPP-NEXT: s_mov_b32 s11, 0
; GFX1032_DPP-NEXT: s_mov_b32 s7, 0x31016000
; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v10, s5
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v9, s4
; GFX1032_DPP-NEXT: s_mov_b32 s4, s2
; GFX1032_DPP-NEXT: s_mov_b32 s5, s3
-; GFX1032_DPP-NEXT: buffer_atomic_sub_x2 v[9:10], off, s[4:7], 0 glc
+; GFX1032_DPP-NEXT: .LBB11_2: ; %atomicrmw.start
+; GFX1032_DPP-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v14, v10
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v13, v9
+; GFX1032_DPP-NEXT: v_sub_co_u32 v11, vcc_lo, v13, s8
+; GFX1032_DPP-NEXT: v_subrev_co_ci_u32_e32 v12, vcc_lo, s9, v14, vcc_lo
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v9, v11
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v10, v12
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v11, v13
+; GFX1032_DPP-NEXT: v_mov_b32_e32 v12, v14
+; GFX1032_DPP-NEXT: buffer_atomic_cmpswap_x2 v[9:12], off, s[4:7], 0 glc
; GFX1032_DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1032_DPP-NEXT: buffer_gl1_inv
; GFX1032_DPP-NEXT: buffer_gl0_inv
-; GFX1032_DPP-NEXT: .LBB11_2:
-; GFX1032_DPP-NEXT: s_waitcnt_depctr 0xffe3
-; GFX1032_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s8
+; GFX1032_DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[9:10], v[13:14]
+; GFX1032_DPP-NEXT: s_or_b32 s11, vcc_lo, s11
+; GFX1032_DPP-NEXT: s_andn2_b32 exec_lo, exec_lo, s11
+; GFX1032_DPP-NEXT: s_cbranch_execnz .LBB11_2
+; GFX1032_DPP-NEXT: ; %bb.3: ; %Flow
+; GFX1032_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s11
+; GFX1032_DPP-NEXT: .LBB11_4: ; %Flow3
+; GFX1032_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s10
; GFX1032_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032_DPP-NEXT: v_readfirstlane_b32 s2, v9
; GFX1032_DPP-NEXT: v_mov_b32_e32 v11, v7
; GFX1032_DPP-NEXT: v_mov_b32_e32 v12, v8
; GFX1032_DPP-NEXT: v_readfirstlane_b32 s3, v10
; GFX1032_DPP-NEXT: v_sub_co_u32 v9, vcc_lo, s2, v11
-; GFX1032_DPP-NEXT: s_mov_b32 s2, s6
+; GFX1032_DPP-NEXT: s_mov_b32 s2, -1
; GFX1032_DPP-NEXT: v_sub_co_ci_u32_e32 v10, vcc_lo, s3, v12, vcc_lo
; GFX1032_DPP-NEXT: s_mov_b32 s3, 0x31016000
; GFX1032_DPP-NEXT: buffer_store_dwordx2 v[9:10], off, s[0:3], 0
@@ -6773,42 +7739,65 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1164_DPP-NEXT: v_readlane_b32 s6, v2, 15
; GFX1164_DPP-NEXT: v_mov_b32_dpp v5, v1 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX1164_DPP-NEXT: v_readlane_b32 s7, v1, 15
-; GFX1164_DPP-NEXT: v_readlane_b32 s8, v2, 31
-; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 31
+; GFX1164_DPP-NEXT: v_readlane_b32 s10, v2, 31
+; GFX1164_DPP-NEXT: v_readlane_b32 s11, v2, 47
; GFX1164_DPP-NEXT: v_writelane_b32 v4, s6, 16
-; GFX1164_DPP-NEXT: v_readlane_b32 s6, v2, 63
+; GFX1164_DPP-NEXT: v_readlane_b32 s6, v1, 31
; GFX1164_DPP-NEXT: v_writelane_b32 v5, s7, 16
-; GFX1164_DPP-NEXT: v_readlane_b32 s10, v2, 47
-; GFX1164_DPP-NEXT: v_readlane_b32 s11, v1, 47
-; GFX1164_DPP-NEXT: v_readlane_b32 s7, v1, 63
-; GFX1164_DPP-NEXT: v_writelane_b32 v4, s8, 32
-; GFX1164_DPP-NEXT: v_writelane_b32 v5, s9, 32
+; GFX1164_DPP-NEXT: v_readlane_b32 s8, v2, 63
+; GFX1164_DPP-NEXT: v_readlane_b32 s9, v1, 63
+; GFX1164_DPP-NEXT: v_readlane_b32 s7, v1, 47
+; GFX1164_DPP-NEXT: v_writelane_b32 v4, s10, 32
+; GFX1164_DPP-NEXT: v_writelane_b32 v5, s6, 32
; GFX1164_DPP-NEXT: s_mov_b64 exec, s[4:5]
; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_DPP-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
-; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[8:9], -1
-; GFX1164_DPP-NEXT: s_mov_b64 s[4:5], s[6:7]
-; GFX1164_DPP-NEXT: v_writelane_b32 v4, s10, 48
-; GFX1164_DPP-NEXT: v_writelane_b32 v5, s11, 48
-; GFX1164_DPP-NEXT: s_mov_b64 exec, s[8:9]
+; GFX1164_DPP-NEXT: s_or_saveexec_b64 s[4:5], -1
+; GFX1164_DPP-NEXT: v_writelane_b32 v4, s11, 48
+; GFX1164_DPP-NEXT: v_writelane_b32 v5, s7, 48
+; GFX1164_DPP-NEXT: s_mov_b64 exec, s[4:5]
; GFX1164_DPP-NEXT: s_mov_b32 s6, -1
-; GFX1164_DPP-NEXT: s_mov_b64 s[8:9], exec
+; GFX1164_DPP-NEXT: s_mov_b64 s[10:11], exec
; GFX1164_DPP-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX1164_DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1164_DPP-NEXT: s_cbranch_execz .LBB11_2
+; GFX1164_DPP-NEXT: s_cbranch_execz .LBB11_4
; GFX1164_DPP-NEXT: ; %bb.1:
-; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, s5
-; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, s4
+; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164_DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
+; GFX1164_DPP-NEXT: s_mov_b64 s[12:13], 0
; GFX1164_DPP-NEXT: s_mov_b32 s7, 0x31016000
; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, s5
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, s4
; GFX1164_DPP-NEXT: s_mov_b32 s4, s2
; GFX1164_DPP-NEXT: s_mov_b32 s5, s3
-; GFX1164_DPP-NEXT: buffer_atomic_sub_u64 v[6:7], off, s[4:7], 0 glc
+; GFX1164_DPP-NEXT: .LBB11_2: ; %atomicrmw.start
+; GFX1164_DPP-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v11, v7
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v10, v6
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164_DPP-NEXT: v_sub_co_u32 v8, vcc, v10, s8
+; GFX1164_DPP-NEXT: v_subrev_co_ci_u32_e64 v9, null, s9, v11, vcc
+; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v6, v8
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v7, v9
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, v10
+; GFX1164_DPP-NEXT: v_mov_b32_e32 v9, v11
+; GFX1164_DPP-NEXT: buffer_atomic_cmpswap_b64 v[6:9], off, s[4:7], 0 glc
; GFX1164_DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164_DPP-NEXT: buffer_gl1_inv
; GFX1164_DPP-NEXT: buffer_gl0_inv
-; GFX1164_DPP-NEXT: .LBB11_2:
-; GFX1164_DPP-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1164_DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[10:11]
+; GFX1164_DPP-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
+; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164_DPP-NEXT: s_and_not1_b64 exec, exec, s[12:13]
+; GFX1164_DPP-NEXT: s_cbranch_execnz .LBB11_2
+; GFX1164_DPP-NEXT: ; %bb.3: ; %Flow
+; GFX1164_DPP-NEXT: s_or_b64 exec, exec, s[12:13]
+; GFX1164_DPP-NEXT: .LBB11_4: ; %Flow3
+; GFX1164_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164_DPP-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX1164_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164_DPP-NEXT: v_readfirstlane_b32 s2, v6
; GFX1164_DPP-NEXT: v_mov_b32_e32 v8, v4
@@ -6816,7 +7805,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1164_DPP-NEXT: v_readfirstlane_b32 s3, v7
; GFX1164_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1164_DPP-NEXT: v_sub_co_u32 v6, vcc, s2, v8
-; GFX1164_DPP-NEXT: s_mov_b32 s2, s6
+; GFX1164_DPP-NEXT: s_mov_b32 s2, -1
; GFX1164_DPP-NEXT: v_sub_co_ci_u32_e64 v7, null, s3, v9, vcc
; GFX1164_DPP-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164_DPP-NEXT: buffer_store_b64 v[6:7], off, s[0:3], 0
@@ -6861,39 +7850,61 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1132_DPP-NEXT: v_mov_b32_dpp v3, v4 quad_perm:[0,1,2,3] row_mask:0xa bank_mask:0xf
; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s0
; GFX1132_DPP-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132_DPP-NEXT: s_or_saveexec_b32 s6, -1
+; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -1
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX1132_DPP-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
-; GFX1132_DPP-NEXT: v_readlane_b32 s4, v2, 31
+; GFX1132_DPP-NEXT: v_readlane_b32 s8, v2, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v6, v2 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX1132_DPP-NEXT: v_readlane_b32 s7, v2, 15
-; GFX1132_DPP-NEXT: v_readlane_b32 s8, v1, 15
-; GFX1132_DPP-NEXT: v_readlane_b32 s5, v1, 31
+; GFX1132_DPP-NEXT: v_readlane_b32 s5, v2, 15
+; GFX1132_DPP-NEXT: v_readlane_b32 s6, v1, 15
+; GFX1132_DPP-NEXT: v_readlane_b32 s9, v1, 31
; GFX1132_DPP-NEXT: v_mov_b32_dpp v7, v1 row_shr:1 row_mask:0xf bank_mask:0xf
-; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s6
+; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4
; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132_DPP-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
-; GFX1132_DPP-NEXT: s_or_saveexec_b32 s6, -1
-; GFX1132_DPP-NEXT: v_writelane_b32 v6, s7, 16
-; GFX1132_DPP-NEXT: v_writelane_b32 v7, s8, 16
-; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s6
+; GFX1132_DPP-NEXT: s_or_saveexec_b32 s4, -1
+; GFX1132_DPP-NEXT: v_writelane_b32 v6, s5, 16
+; GFX1132_DPP-NEXT: v_writelane_b32 v7, s6, 16
+; GFX1132_DPP-NEXT: s_mov_b32 exec_lo, s4
; GFX1132_DPP-NEXT: s_mov_b32 s6, -1
-; GFX1132_DPP-NEXT: s_mov_b32 s8, exec_lo
+; GFX1132_DPP-NEXT: s_mov_b32 s10, exec_lo
; GFX1132_DPP-NEXT: ; implicit-def: $vgpr8_vgpr9
; GFX1132_DPP-NEXT: v_cmpx_eq_u32_e32 0, v0
-; GFX1132_DPP-NEXT: s_cbranch_execz .LBB11_2
+; GFX1132_DPP-NEXT: s_cbranch_execz .LBB11_4
; GFX1132_DPP-NEXT: ; %bb.1:
-; GFX1132_DPP-NEXT: v_dual_mov_b32 v9, s5 :: v_dual_mov_b32 v8, s4
+; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132_DPP-NEXT: s_load_b64 s[4:5], s[2:3], 0x0
+; GFX1132_DPP-NEXT: s_mov_b32 s11, 0
; GFX1132_DPP-NEXT: s_mov_b32 s7, 0x31016000
; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v9, s5 :: v_dual_mov_b32 v8, s4
; GFX1132_DPP-NEXT: s_mov_b32 s4, s2
; GFX1132_DPP-NEXT: s_mov_b32 s5, s3
-; GFX1132_DPP-NEXT: buffer_atomic_sub_u64 v[8:9], off, s[4:7], 0 glc
+; GFX1132_DPP-NEXT: .LBB11_2: ; %atomicrmw.start
+; GFX1132_DPP-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v13, v9 :: v_dual_mov_b32 v12, v8
+; GFX1132_DPP-NEXT: v_sub_co_u32 v10, vcc_lo, v12, s8
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1132_DPP-NEXT: v_subrev_co_ci_u32_e64 v11, null, s9, v13, vcc_lo
+; GFX1132_DPP-NEXT: v_mov_b32_e32 v8, v10
+; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1132_DPP-NEXT: v_dual_mov_b32 v9, v11 :: v_dual_mov_b32 v10, v12
+; GFX1132_DPP-NEXT: v_mov_b32_e32 v11, v13
+; GFX1132_DPP-NEXT: buffer_atomic_cmpswap_b64 v[8:11], off, s[4:7], 0 glc
; GFX1132_DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132_DPP-NEXT: buffer_gl1_inv
; GFX1132_DPP-NEXT: buffer_gl0_inv
-; GFX1132_DPP-NEXT: .LBB11_2:
-; GFX1132_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s8
+; GFX1132_DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[8:9], v[12:13]
+; GFX1132_DPP-NEXT: s_or_b32 s11, vcc_lo, s11
+; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_and_not1_b32 exec_lo, exec_lo, s11
+; GFX1132_DPP-NEXT: s_cbranch_execnz .LBB11_2
+; GFX1132_DPP-NEXT: ; %bb.3: ; %Flow
+; GFX1132_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s11
+; GFX1132_DPP-NEXT: .LBB11_4: ; %Flow3
+; GFX1132_DPP-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132_DPP-NEXT: s_or_b32 exec_lo, exec_lo, s10
; GFX1132_DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132_DPP-NEXT: v_readfirstlane_b32 s2, v8
; GFX1132_DPP-NEXT: v_mov_b32_e32 v10, v6
@@ -6901,7 +7912,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1132_DPP-NEXT: v_readfirstlane_b32 s3, v9
; GFX1132_DPP-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1132_DPP-NEXT: v_sub_co_u32 v8, vcc_lo, s2, v10
-; GFX1132_DPP-NEXT: s_mov_b32 s2, s6
+; GFX1132_DPP-NEXT: s_mov_b32 s2, -1
; GFX1132_DPP-NEXT: v_sub_co_ci_u32_e64 v9, null, s3, v11, vcc_lo
; GFX1132_DPP-NEXT: s_mov_b32 s3, 0x31016000
; GFX1132_DPP-NEXT: buffer_store_b64 v[8:9], off, s[0:3], 0
@@ -7111,7 +8122,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
entry:
%lane = call i32 @llvm.amdgcn.workitem.id.x()
%zext = zext i32 %lane to i64
- %old = atomicrmw sub ptr addrspace(1) %inout, i64 %zext syncscope("agent") acq_rel, !amdgpu.no.fine.grained.memory !0
+ %old = atomicrmw sub ptr addrspace(1) %inout, i64 %zext syncscope("agent") acq_rel
store i64 %old, ptr addrspace(1) %out
ret void
}
@@ -7119,504 +8130,716 @@ entry:
define amdgpu_kernel void @uniform_or_i8(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i8 %val) {
; GFX7LESS-LABEL: uniform_or_i8:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xd
+; GFX7LESS-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX7LESS-NEXT: s_load_dword s12, s[4:5], 0xd
; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX7LESS-NEXT: s_cbranch_execz .LBB12_2
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT: s_cbranch_execz .LBB12_4
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_and_b32 s8, s2, -4
+; GFX7LESS-NEXT: s_and_b32 s4, s10, -4
+; GFX7LESS-NEXT: s_mov_b32 s5, s11
+; GFX7LESS-NEXT: s_and_b32 s0, s10, 3
+; GFX7LESS-NEXT: s_and_b32 s1, s12, 0xff
+; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0x0
+; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_lshl_b32 s13, s0, 3
+; GFX7LESS-NEXT: s_lshl_b32 s14, s1, s13
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: .LBB12_2: ; %atomicrmw.start
+; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX7LESS-NEXT: s_waitcnt expcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
+; GFX7LESS-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX7LESS-NEXT: s_cbranch_execnz .LBB12_2
+; GFX7LESS-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX7LESS-NEXT: .LBB12_4: ; %Flow
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_mov_b32 s11, 0xf000
-; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
-; GFX7LESS-NEXT: s_lshl_b32 s2, s2, 3
-; GFX7LESS-NEXT: s_and_b32 s7, s6, 0xff
-; GFX7LESS-NEXT: s_lshl_b32 s7, s7, s2
; GFX7LESS-NEXT: s_mov_b32 s10, -1
-; GFX7LESS-NEXT: s_mov_b32 s9, s3
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s7
-; GFX7LESS-NEXT: buffer_atomic_or v0, off, s[8:11], 0 glc
-; GFX7LESS-NEXT: s_waitcnt vmcnt(0) expcnt(0)
-; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s2, v0
-; GFX7LESS-NEXT: .LBB12_2:
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
-; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v0
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s12
+; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v0
; GFX7LESS-NEXT: v_cndmask_b32_e64 v0, v1, 0, vcc
-; GFX7LESS-NEXT: v_or_b32_e32 v0, s4, v0
-; GFX7LESS-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX7LESS-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX7LESS-NEXT: buffer_store_byte v0, off, s[8:11], 0
; GFX7LESS-NEXT: s_endpgm
;
; GFX8-LABEL: uniform_or_i8:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX8-NEXT: s_load_dword s12, s[4:5], 0x34
; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr0
-; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX8-NEXT: s_cbranch_execz .LBB12_2
+; GFX8-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX8-NEXT: s_cbranch_execz .LBB12_4
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s8, s2, -4
-; GFX8-NEXT: s_and_b32 s2, s2, 3
-; GFX8-NEXT: s_mov_b32 s9, s3
-; GFX8-NEXT: s_lshl_b32 s2, s2, 3
-; GFX8-NEXT: s_and_b32 s3, s6, 0xff
-; GFX8-NEXT: s_lshl_b32 s3, s3, s2
-; GFX8-NEXT: s_mov_b32 s11, 0xf000
-; GFX8-NEXT: s_mov_b32 s10, -1
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: buffer_atomic_or v0, off, s[8:11], 0 glc
+; GFX8-NEXT: s_and_b32 s4, s10, -4
+; GFX8-NEXT: s_mov_b32 s5, s11
+; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0
+; GFX8-NEXT: s_and_b32 s0, s10, 3
+; GFX8-NEXT: s_lshl_b32 s13, s0, 3
+; GFX8-NEXT: s_and_b32 s0, s12, 0xff
+; GFX8-NEXT: s_lshl_b32 s14, s0, s13
+; GFX8-NEXT: s_mov_b64 s[10:11], 0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: .LBB12_2: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX8-NEXT: v_mov_b32_e32 v3, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, s2, v0
-; GFX8-NEXT: .LBB12_2:
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
+; GFX8-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX8-NEXT: v_mov_b32_e32 v1, v2
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX8-NEXT: s_cbranch_execnz .LBB12_2
+; GFX8-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX8-NEXT: .LBB12_4: ; %Flow
+; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX8-NEXT: v_readfirstlane_b32 s4, v0
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: v_mov_b32_e32 v0, s12
; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
-; GFX8-NEXT: v_or_b32_e32 v0, s4, v0
-; GFX8-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX8-NEXT: s_mov_b32 s11, 0xf000
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT: buffer_store_byte v0, off, s[8:11], 0
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: uniform_or_i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s12, s[4:5], 0x34
; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr0
-; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX9-NEXT: s_cbranch_execz .LBB12_2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT: s_cbranch_execz .LBB12_4
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s8, s2, -4
-; GFX9-NEXT: s_and_b32 s2, s2, 3
-; GFX9-NEXT: s_mov_b32 s9, s3
-; GFX9-NEXT: s_lshl_b32 s2, s2, 3
-; GFX9-NEXT: s_and_b32 s3, s6, 0xff
-; GFX9-NEXT: s_lshl_b32 s3, s3, s2
-; GFX9-NEXT: s_mov_b32 s11, 0xf000
-; GFX9-NEXT: s_mov_b32 s10, -1
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: buffer_atomic_or v0, off, s[8:11], 0 glc
+; GFX9-NEXT: s_and_b32 s4, s10, -4
+; GFX9-NEXT: s_mov_b32 s5, s11
+; GFX9-NEXT: s_load_dword s1, s[4:5], 0x0
+; GFX9-NEXT: s_and_b32 s0, s10, 3
+; GFX9-NEXT: s_lshl_b32 s13, s0, 3
+; GFX9-NEXT: s_and_b32 s0, s12, 0xff
+; GFX9-NEXT: s_lshl_b32 s14, s0, s13
+; GFX9-NEXT: s_mov_b64 s[10:11], 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: .LBB12_2: ; %atomicrmw.start
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, s2, v0
-; GFX9-NEXT: .LBB12_2:
-; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
+; GFX9-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX9-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX9-NEXT: s_cbranch_execnz .LBB12_2
+; GFX9-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX9-NEXT: .LBB12_4: ; %Flow
+; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX9-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX9-NEXT: v_readfirstlane_b32 s4, v0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-NEXT: v_mov_b32_e32 v0, s12
; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
-; GFX9-NEXT: s_mov_b32 s3, 0xf000
-; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: v_or_b32_e32 v0, s4, v0
-; GFX9-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX9-NEXT: s_mov_b32 s11, 0xf000
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT: buffer_store_byte v0, off, s[8:11], 0
; GFX9-NEXT: s_endpgm
;
; GFX1064-LABEL: uniform_or_i8:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_clause 0x1
-; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX1064-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX1064-NEXT: s_load_dword s12, s[4:5], 0x34
; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0
-; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX1064-NEXT: s_cbranch_execz .LBB12_2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT: s_cbranch_execz .LBB12_4
; GFX1064-NEXT: ; %bb.1:
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_and_b32 s7, s2, 3
-; GFX1064-NEXT: s_and_b32 s8, s6, 0xff
-; GFX1064-NEXT: s_lshl_b32 s7, s7, 3
-; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1064-NEXT: s_lshl_b32 s9, s8, s7
-; GFX1064-NEXT: s_and_b32 s8, s2, -4
-; GFX1064-NEXT: v_mov_b32_e32 v0, s9
-; GFX1064-NEXT: s_mov_b32 s10, -1
-; GFX1064-NEXT: s_mov_b32 s9, s3
-; GFX1064-NEXT: buffer_atomic_or v0, off, s[8:11], 0 glc
+; GFX1064-NEXT: s_and_b32 s4, s10, -4
+; GFX1064-NEXT: s_mov_b32 s5, s11
+; GFX1064-NEXT: s_and_b32 s1, s10, 3
+; GFX1064-NEXT: s_load_dword s0, s[4:5], 0x0
+; GFX1064-NEXT: s_lshl_b32 s13, s1, 3
+; GFX1064-NEXT: s_and_b32 s1, s12, 0xff
+; GFX1064-NEXT: s_mov_b64 s[10:11], 0
+; GFX1064-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT: s_mov_b32 s6, -1
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: v_mov_b32_e32 v1, s0
+; GFX1064-NEXT: .LBB12_2: ; %atomicrmw.start
+; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1064-NEXT: v_mov_b32_e32 v3, v1
+; GFX1064-NEXT: v_mov_b32_e32 v2, v0
+; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s7, v0
-; GFX1064-NEXT: .LBB12_2:
-; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
+; GFX1064-NEXT: v_mov_b32_e32 v1, v2
+; GFX1064-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX1064-NEXT: s_cbranch_execnz .LBB12_2
+; GFX1064-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1064-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1064-NEXT: .LBB12_4: ; %Flow
+; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1064-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1064-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1064-NEXT: v_cndmask_b32_e64 v0, s6, 0, vcc
-; GFX1064-NEXT: v_or_b32_e32 v0, s2, v0
-; GFX1064-NEXT: s_mov_b32 s2, -1
-; GFX1064-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1064-NEXT: s_mov_b32 s10, -1
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-NEXT: v_cndmask_b32_e64 v0, s12, 0, vcc
+; GFX1064-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX1064-NEXT: buffer_store_byte v0, off, s[8:11], 0
; GFX1064-NEXT: s_endpgm
;
; GFX1032-LABEL: uniform_or_i8:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_clause 0x1
-; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1032-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX1032-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX1032-NEXT: s_load_dword s1, s[4:5], 0x34
; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s3, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: ; implicit-def: $vgpr0
-; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo
-; GFX1032-NEXT: s_cbranch_execz .LBB12_2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
+; GFX1032-NEXT: s_cbranch_execz .LBB12_4
; GFX1032-NEXT: ; %bb.1:
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_and_b32 s5, s2, 3
-; GFX1032-NEXT: s_and_b32 s7, s6, 0xff
-; GFX1032-NEXT: s_lshl_b32 s5, s5, 3
-; GFX1032-NEXT: s_and_b32 s8, s2, -4
-; GFX1032-NEXT: s_lshl_b32 s7, s7, s5
-; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1032-NEXT: v_mov_b32_e32 v0, s7
-; GFX1032-NEXT: s_mov_b32 s10, -1
-; GFX1032-NEXT: s_mov_b32 s9, s3
-; GFX1032-NEXT: buffer_atomic_or v0, off, s[8:11], 0 glc
+; GFX1032-NEXT: s_and_b32 s4, s10, -4
+; GFX1032-NEXT: s_mov_b32 s5, s11
+; GFX1032-NEXT: s_and_b32 s6, s10, 3
+; GFX1032-NEXT: s_load_dword s0, s[4:5], 0x0
+; GFX1032-NEXT: s_lshl_b32 s10, s6, 3
+; GFX1032-NEXT: s_and_b32 s6, s1, 0xff
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1032-NEXT: s_mov_b32 s6, -1
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v1, s0
+; GFX1032-NEXT: .LBB12_2: ; %atomicrmw.start
+; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1032-NEXT: v_mov_b32_e32 v3, v1
+; GFX1032-NEXT: v_mov_b32_e32 v2, v0
+; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s5, v0
-; GFX1032-NEXT: .LBB12_2:
-; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT: v_cmp_eq_u32_e64 s0, v2, v1
+; GFX1032-NEXT: v_mov_b32_e32 v1, v2
+; GFX1032-NEXT: s_or_b32 s3, s0, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
+; GFX1032-NEXT: s_cbranch_execnz .LBB12_2
+; GFX1032-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s10, v2
+; GFX1032-NEXT: .LBB12_4: ; %Flow
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1032-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1032-NEXT: v_cndmask_b32_e64 v0, s6, 0, vcc_lo
-; GFX1032-NEXT: v_or_b32_e32 v0, s2, v0
-; GFX1032-NEXT: s_mov_b32 s2, -1
-; GFX1032-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1032-NEXT: s_mov_b32 s10, -1
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-NEXT: v_cndmask_b32_e64 v0, s1, 0, vcc_lo
+; GFX1032-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX1032-NEXT: buffer_store_byte v0, off, s[8:11], 0
; GFX1032-NEXT: s_endpgm
;
; GFX1164-TRUE16-LABEL: uniform_or_i8:
; GFX1164-TRUE16: ; %bb.0:
; GFX1164-TRUE16-NEXT: s_clause 0x1
-; GFX1164-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1164-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
+; GFX1164-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1164-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1164-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX1164-TRUE16-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX1164-TRUE16-NEXT: s_cbranch_execz .LBB12_2
+; GFX1164-TRUE16-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1164-TRUE16-NEXT: s_cbranch_execz .LBB12_4
; GFX1164-TRUE16-NEXT: ; %bb.1:
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-TRUE16-NEXT: s_and_b32 s7, s2, 3
-; GFX1164-TRUE16-NEXT: s_and_b32 s8, s6, 0xff
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s7, s7, 3
-; GFX1164-TRUE16-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s9, s8, s7
-; GFX1164-TRUE16-NEXT: s_and_b32 s8, s2, -4
-; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v0, s9
-; GFX1164-TRUE16-NEXT: s_mov_b32 s10, -1
-; GFX1164-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX1164-TRUE16-NEXT: buffer_atomic_or_b32 v0, off, s[8:11], 0 glc
+; GFX1164-TRUE16-NEXT: s_and_b32 s4, s10, -4
+; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s11
+; GFX1164-TRUE16-NEXT: s_and_b32 s1, s10, 3
+; GFX1164-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s13, s1, 3
+; GFX1164-TRUE16-NEXT: s_and_b32 s1, s12, 0xff
+; GFX1164-TRUE16-NEXT: s_mov_b64 s[10:11], 0
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s0
+; GFX1164-TRUE16-NEXT: .LBB12_2: ; %atomicrmw.start
+; GFX1164-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-TRUE16-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX1164-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s7, v0
-; GFX1164-TRUE16-NEXT: .LBB12_2:
-; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
+; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1164-TRUE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1164-TRUE16-NEXT: s_cbranch_execnz .LBB12_2
+; GFX1164-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1164-TRUE16-NEXT: .LBB12_4: ; %Flow
+; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1164-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1164-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1164-TRUE16-NEXT: v_cndmask_b16 v0.l, s6, 0, vcc
+; GFX1164-TRUE16-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1164-TRUE16-NEXT: s_mov_b32 s10, -1
+; GFX1164-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-TRUE16-NEXT: v_cndmask_b16 v0.l, s12, 0, vcc
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-TRUE16-NEXT: v_or_b16 v0.l, s2, v0.l
-; GFX1164-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX1164-TRUE16-NEXT: buffer_store_b8 v0, off, s[0:3], 0
+; GFX1164-TRUE16-NEXT: v_or_b16 v0.l, s0, v0.l
+; GFX1164-TRUE16-NEXT: buffer_store_b8 v0, off, s[8:11], 0
; GFX1164-TRUE16-NEXT: s_endpgm
;
; GFX1164-FAKE16-LABEL: uniform_or_i8:
; GFX1164-FAKE16: ; %bb.0:
; GFX1164-FAKE16-NEXT: s_clause 0x1
-; GFX1164-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1164-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
+; GFX1164-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1164-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1164-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX1164-FAKE16-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX1164-FAKE16-NEXT: s_cbranch_execz .LBB12_2
+; GFX1164-FAKE16-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1164-FAKE16-NEXT: s_cbranch_execz .LBB12_4
; GFX1164-FAKE16-NEXT: ; %bb.1:
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-FAKE16-NEXT: s_and_b32 s7, s2, 3
-; GFX1164-FAKE16-NEXT: s_and_b32 s8, s6, 0xff
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s7, s7, 3
-; GFX1164-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s9, s8, s7
-; GFX1164-FAKE16-NEXT: s_and_b32 s8, s2, -4
-; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v0, s9
-; GFX1164-FAKE16-NEXT: s_mov_b32 s10, -1
-; GFX1164-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX1164-FAKE16-NEXT: buffer_atomic_or_b32 v0, off, s[8:11], 0 glc
+; GFX1164-FAKE16-NEXT: s_and_b32 s4, s10, -4
+; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s11
+; GFX1164-FAKE16-NEXT: s_and_b32 s1, s10, 3
+; GFX1164-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s13, s1, 3
+; GFX1164-FAKE16-NEXT: s_and_b32 s1, s12, 0xff
+; GFX1164-FAKE16-NEXT: s_mov_b64 s[10:11], 0
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s0
+; GFX1164-FAKE16-NEXT: .LBB12_2: ; %atomicrmw.start
+; GFX1164-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-FAKE16-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX1164-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s7, v0
-; GFX1164-FAKE16-NEXT: .LBB12_2:
-; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
+; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1164-FAKE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1164-FAKE16-NEXT: s_cbranch_execnz .LBB12_2
+; GFX1164-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1164-FAKE16-NEXT: .LBB12_4: ; %Flow
+; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1164-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1164-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1164-FAKE16-NEXT: v_cndmask_b32_e64 v0, s6, 0, vcc
+; GFX1164-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1164-FAKE16-NEXT: s_mov_b32 s10, -1
+; GFX1164-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-FAKE16-NEXT: v_cndmask_b32_e64 v0, s12, 0, vcc
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-FAKE16-NEXT: v_or_b32_e32 v0, s2, v0
-; GFX1164-FAKE16-NEXT: s_mov_b32 s2, -1
-; GFX1164-FAKE16-NEXT: buffer_store_b8 v0, off, s[0:3], 0
+; GFX1164-FAKE16-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX1164-FAKE16-NEXT: buffer_store_b8 v0, off, s[8:11], 0
; GFX1164-FAKE16-NEXT: s_endpgm
;
; GFX1132-TRUE16-LABEL: uniform_or_i8:
; GFX1132-TRUE16: ; %bb.0:
; GFX1132-TRUE16-NEXT: s_clause 0x1
-; GFX1132-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132-TRUE16-NEXT: s_load_b32 s4, s[4:5], 0x34
+; GFX1132-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
+; GFX1132-TRUE16-NEXT: s_load_b32 s1, s[4:5], 0x34
; GFX1132-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-TRUE16-NEXT: s_mov_b32 s3, 0
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1132-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX1132-TRUE16-NEXT: s_and_saveexec_b32 s5, vcc_lo
-; GFX1132-TRUE16-NEXT: s_cbranch_execz .LBB12_2
+; GFX1132-TRUE16-NEXT: s_and_saveexec_b32 s2, vcc_lo
+; GFX1132-TRUE16-NEXT: s_cbranch_execz .LBB12_4
; GFX1132-TRUE16-NEXT: ; %bb.1:
; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-TRUE16-NEXT: s_and_b32 s6, s2, 3
-; GFX1132-TRUE16-NEXT: s_and_b32 s7, s4, 0xff
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s6, s6, 3
-; GFX1132-TRUE16-NEXT: s_and_b32 s8, s2, -4
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s7, s7, s6
-; GFX1132-TRUE16-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v0, s7
-; GFX1132-TRUE16-NEXT: s_mov_b32 s10, -1
-; GFX1132-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX1132-TRUE16-NEXT: buffer_atomic_or_b32 v0, off, s[8:11], 0 glc
+; GFX1132-TRUE16-NEXT: s_and_b32 s4, s10, -4
+; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s11
+; GFX1132-TRUE16-NEXT: s_and_b32 s6, s10, 3
+; GFX1132-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s10, s6, 3
+; GFX1132-TRUE16-NEXT: s_and_b32 s6, s1, 0xff
+; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s0
+; GFX1132-TRUE16-NEXT: .LBB12_2: ; %atomicrmw.start
+; GFX1132-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-TRUE16-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1132-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s6, v0
-; GFX1132-TRUE16-NEXT: .LBB12_2:
-; GFX1132-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v1
+; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1132-TRUE16-NEXT: s_or_b32 s3, s0, s3
+; GFX1132-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
+; GFX1132-TRUE16-NEXT: s_cbranch_execnz .LBB12_2
+; GFX1132-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1132-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1132-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s10, v2
+; GFX1132-TRUE16-NEXT: .LBB12_4: ; %Flow
+; GFX1132-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s2
+; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1132-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1132-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1132-TRUE16-NEXT: v_cndmask_b16 v0.l, s4, 0, vcc_lo
+; GFX1132-TRUE16-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1132-TRUE16-NEXT: s_mov_b32 s10, -1
+; GFX1132-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-TRUE16-NEXT: v_cndmask_b16 v0.l, s1, 0, vcc_lo
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT: v_or_b16 v0.l, s2, v0.l
-; GFX1132-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX1132-TRUE16-NEXT: buffer_store_b8 v0, off, s[0:3], 0
+; GFX1132-TRUE16-NEXT: v_or_b16 v0.l, s0, v0.l
+; GFX1132-TRUE16-NEXT: buffer_store_b8 v0, off, s[8:11], 0
; GFX1132-TRUE16-NEXT: s_endpgm
;
; GFX1132-FAKE16-LABEL: uniform_or_i8:
; GFX1132-FAKE16: ; %bb.0:
; GFX1132-FAKE16-NEXT: s_clause 0x1
-; GFX1132-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132-FAKE16-NEXT: s_load_b32 s4, s[4:5], 0x34
+; GFX1132-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
+; GFX1132-FAKE16-NEXT: s_load_b32 s1, s[4:5], 0x34
; GFX1132-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-FAKE16-NEXT: s_mov_b32 s3, 0
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1132-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX1132-FAKE16-NEXT: s_and_saveexec_b32 s5, vcc_lo
-; GFX1132-FAKE16-NEXT: s_cbranch_execz .LBB12_2
+; GFX1132-FAKE16-NEXT: s_and_saveexec_b32 s2, vcc_lo
+; GFX1132-FAKE16-NEXT: s_cbranch_execz .LBB12_4
; GFX1132-FAKE16-NEXT: ; %bb.1:
; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-FAKE16-NEXT: s_and_b32 s6, s2, 3
-; GFX1132-FAKE16-NEXT: s_and_b32 s7, s4, 0xff
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s6, s6, 3
-; GFX1132-FAKE16-NEXT: s_and_b32 s8, s2, -4
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s7, s7, s6
-; GFX1132-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v0, s7
-; GFX1132-FAKE16-NEXT: s_mov_b32 s10, -1
-; GFX1132-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX1132-FAKE16-NEXT: buffer_atomic_or_b32 v0, off, s[8:11], 0 glc
+; GFX1132-FAKE16-NEXT: s_and_b32 s4, s10, -4
+; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s11
+; GFX1132-FAKE16-NEXT: s_and_b32 s6, s10, 3
+; GFX1132-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s10, s6, 3
+; GFX1132-FAKE16-NEXT: s_and_b32 s6, s1, 0xff
+; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s0
+; GFX1132-FAKE16-NEXT: .LBB12_2: ; %atomicrmw.start
+; GFX1132-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-FAKE16-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1132-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s6, v0
-; GFX1132-FAKE16-NEXT: .LBB12_2:
-; GFX1132-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-FAKE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v1
+; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1132-FAKE16-NEXT: s_or_b32 s3, s0, s3
+; GFX1132-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
+; GFX1132-FAKE16-NEXT: s_cbranch_execnz .LBB12_2
+; GFX1132-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1132-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1132-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s10, v2
+; GFX1132-FAKE16-NEXT: .LBB12_4: ; %Flow
+; GFX1132-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s2
+; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1132-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1132-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1132-FAKE16-NEXT: v_cndmask_b32_e64 v0, s4, 0, vcc_lo
+; GFX1132-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1132-FAKE16-NEXT: s_mov_b32 s10, -1
+; GFX1132-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-FAKE16-NEXT: v_cndmask_b32_e64 v0, s1, 0, vcc_lo
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-FAKE16-NEXT: v_or_b32_e32 v0, s2, v0
-; GFX1132-FAKE16-NEXT: s_mov_b32 s2, -1
-; GFX1132-FAKE16-NEXT: buffer_store_b8 v0, off, s[0:3], 0
+; GFX1132-FAKE16-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX1132-FAKE16-NEXT: buffer_store_b8 v0, off, s[8:11], 0
; GFX1132-FAKE16-NEXT: s_endpgm
;
; GFX1264-TRUE16-LABEL: uniform_or_i8:
; GFX1264-TRUE16: ; %bb.0:
; GFX1264-TRUE16-NEXT: s_clause 0x1
-; GFX1264-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1264-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
+; GFX1264-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1264-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1264-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1264-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX1264-TRUE16-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX1264-TRUE16-NEXT: s_cbranch_execz .LBB12_2
+; GFX1264-TRUE16-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1264-TRUE16-NEXT: s_cbranch_execz .LBB12_4
; GFX1264-TRUE16-NEXT: ; %bb.1:
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-TRUE16-NEXT: s_and_b32 s7, s2, 3
-; GFX1264-TRUE16-NEXT: s_and_b32 s8, s6, 0xff
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s7, s7, 3
-; GFX1264-TRUE16-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s9, s8, s7
-; GFX1264-TRUE16-NEXT: s_and_b32 s8, s2, -4
-; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v0, s9
-; GFX1264-TRUE16-NEXT: s_mov_b32 s10, -1
-; GFX1264-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX1264-TRUE16-NEXT: buffer_atomic_or_b32 v0, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1264-TRUE16-NEXT: s_and_b32 s4, s10, -4
+; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s11
+; GFX1264-TRUE16-NEXT: s_and_b32 s1, s10, 3
+; GFX1264-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s13, s1, 3
+; GFX1264-TRUE16-NEXT: s_and_b32 s1, s12, 0xff
+; GFX1264-TRUE16-NEXT: s_mov_b64 s[10:11], 0
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s0
+; GFX1264-TRUE16-NEXT: .LBB12_2: ; %atomicrmw.start
+; GFX1264-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1264-TRUE16-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX1264-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s7, v0
-; GFX1264-TRUE16-NEXT: .LBB12_2:
-; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
+; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1264-TRUE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1264-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1264-TRUE16-NEXT: s_cbranch_execnz .LBB12_2
+; GFX1264-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1264-TRUE16-NEXT: .LBB12_4: ; %Flow
+; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1264-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1264-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1264-TRUE16-NEXT: v_cndmask_b16 v0.l, s6, 0, vcc
+; GFX1264-TRUE16-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1264-TRUE16-NEXT: s_mov_b32 s10, -1
+; GFX1264-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1264-TRUE16-NEXT: v_cndmask_b16 v0.l, s12, 0, vcc
+; GFX1264-TRUE16-NEXT: s_wait_alu 0xf1ff
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1264-TRUE16-NEXT: v_or_b16 v0.l, s2, v0.l
-; GFX1264-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX1264-TRUE16-NEXT: buffer_store_b8 v0, off, s[0:3], null
+; GFX1264-TRUE16-NEXT: v_or_b16 v0.l, s0, v0.l
+; GFX1264-TRUE16-NEXT: buffer_store_b8 v0, off, s[8:11], null
; GFX1264-TRUE16-NEXT: s_endpgm
;
; GFX1264-FAKE16-LABEL: uniform_or_i8:
; GFX1264-FAKE16: ; %bb.0:
; GFX1264-FAKE16-NEXT: s_clause 0x1
-; GFX1264-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1264-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
+; GFX1264-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1264-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1264-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1264-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX1264-FAKE16-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX1264-FAKE16-NEXT: s_cbranch_execz .LBB12_2
+; GFX1264-FAKE16-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1264-FAKE16-NEXT: s_cbranch_execz .LBB12_4
; GFX1264-FAKE16-NEXT: ; %bb.1:
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-FAKE16-NEXT: s_and_b32 s7, s2, 3
-; GFX1264-FAKE16-NEXT: s_and_b32 s8, s6, 0xff
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s7, s7, 3
-; GFX1264-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s9, s8, s7
-; GFX1264-FAKE16-NEXT: s_and_b32 s8, s2, -4
-; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v0, s9
-; GFX1264-FAKE16-NEXT: s_mov_b32 s10, -1
-; GFX1264-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX1264-FAKE16-NEXT: buffer_atomic_or_b32 v0, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1264-FAKE16-NEXT: s_and_b32 s4, s10, -4
+; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s11
+; GFX1264-FAKE16-NEXT: s_and_b32 s1, s10, 3
+; GFX1264-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s13, s1, 3
+; GFX1264-FAKE16-NEXT: s_and_b32 s1, s12, 0xff
+; GFX1264-FAKE16-NEXT: s_mov_b64 s[10:11], 0
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s0
+; GFX1264-FAKE16-NEXT: .LBB12_2: ; %atomicrmw.start
+; GFX1264-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1264-FAKE16-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX1264-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s7, v0
-; GFX1264-FAKE16-NEXT: .LBB12_2:
-; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
+; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1264-FAKE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1264-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1264-FAKE16-NEXT: s_cbranch_execnz .LBB12_2
+; GFX1264-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1264-FAKE16-NEXT: .LBB12_4: ; %Flow
+; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1264-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1264-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1264-FAKE16-NEXT: v_cndmask_b32_e64 v0, s6, 0, vcc
+; GFX1264-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1264-FAKE16-NEXT: s_mov_b32 s10, -1
+; GFX1264-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1264-FAKE16-NEXT: v_cndmask_b32_e64 v0, s12, 0, vcc
+; GFX1264-FAKE16-NEXT: s_wait_alu 0xf1ff
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1264-FAKE16-NEXT: v_or_b32_e32 v0, s2, v0
-; GFX1264-FAKE16-NEXT: s_mov_b32 s2, -1
-; GFX1264-FAKE16-NEXT: buffer_store_b8 v0, off, s[0:3], null
+; GFX1264-FAKE16-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX1264-FAKE16-NEXT: buffer_store_b8 v0, off, s[8:11], null
; GFX1264-FAKE16-NEXT: s_endpgm
;
; GFX1232-TRUE16-LABEL: uniform_or_i8:
; GFX1232-TRUE16: ; %bb.0:
; GFX1232-TRUE16-NEXT: s_clause 0x1
-; GFX1232-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1232-TRUE16-NEXT: s_load_b32 s4, s[4:5], 0x34
+; GFX1232-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
+; GFX1232-TRUE16-NEXT: s_load_b32 s1, s[4:5], 0x34
; GFX1232-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1232-TRUE16-NEXT: s_mov_b32 s3, 0
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1232-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1232-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX1232-TRUE16-NEXT: s_and_saveexec_b32 s5, vcc_lo
-; GFX1232-TRUE16-NEXT: s_cbranch_execz .LBB12_2
+; GFX1232-TRUE16-NEXT: s_and_saveexec_b32 s2, vcc_lo
+; GFX1232-TRUE16-NEXT: s_cbranch_execz .LBB12_4
; GFX1232-TRUE16-NEXT: ; %bb.1:
; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1232-TRUE16-NEXT: s_and_b32 s6, s2, 3
-; GFX1232-TRUE16-NEXT: s_and_b32 s7, s4, 0xff
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s6, s6, 3
-; GFX1232-TRUE16-NEXT: s_and_b32 s8, s2, -4
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s7, s7, s6
-; GFX1232-TRUE16-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v0, s7
-; GFX1232-TRUE16-NEXT: s_mov_b32 s10, -1
-; GFX1232-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX1232-TRUE16-NEXT: buffer_atomic_or_b32 v0, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1232-TRUE16-NEXT: s_and_b32 s4, s10, -4
+; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s11
+; GFX1232-TRUE16-NEXT: s_and_b32 s6, s10, 3
+; GFX1232-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s10, s6, 3
+; GFX1232-TRUE16-NEXT: s_and_b32 s6, s1, 0xff
+; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s0
+; GFX1232-TRUE16-NEXT: .LBB12_2: ; %atomicrmw.start
+; GFX1232-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-TRUE16-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1232-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1232-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1232-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s6, v0
-; GFX1232-TRUE16-NEXT: .LBB12_2:
-; GFX1232-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1232-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v1
+; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1232-TRUE16-NEXT: s_or_b32 s3, s0, s3
+; GFX1232-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1232-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
+; GFX1232-TRUE16-NEXT: s_cbranch_execnz .LBB12_2
+; GFX1232-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1232-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1232-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s10, v2
+; GFX1232-TRUE16-NEXT: .LBB12_4: ; %Flow
+; GFX1232-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s2
+; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1232-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1232-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1232-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1232-TRUE16-NEXT: v_cndmask_b16 v0.l, s4, 0, vcc_lo
+; GFX1232-TRUE16-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1232-TRUE16-NEXT: s_mov_b32 s10, -1
+; GFX1232-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1232-TRUE16-NEXT: v_cndmask_b16 v0.l, s1, 0, vcc_lo
+; GFX1232-TRUE16-NEXT: s_wait_alu 0xf1ff
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT: v_or_b16 v0.l, s2, v0.l
-; GFX1232-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX1232-TRUE16-NEXT: buffer_store_b8 v0, off, s[0:3], null
+; GFX1232-TRUE16-NEXT: v_or_b16 v0.l, s0, v0.l
+; GFX1232-TRUE16-NEXT: buffer_store_b8 v0, off, s[8:11], null
; GFX1232-TRUE16-NEXT: s_endpgm
;
; GFX1232-FAKE16-LABEL: uniform_or_i8:
; GFX1232-FAKE16: ; %bb.0:
; GFX1232-FAKE16-NEXT: s_clause 0x1
-; GFX1232-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1232-FAKE16-NEXT: s_load_b32 s4, s[4:5], 0x34
+; GFX1232-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
+; GFX1232-FAKE16-NEXT: s_load_b32 s1, s[4:5], 0x34
; GFX1232-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1232-FAKE16-NEXT: s_mov_b32 s3, 0
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1232-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1232-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX1232-FAKE16-NEXT: s_and_saveexec_b32 s5, vcc_lo
-; GFX1232-FAKE16-NEXT: s_cbranch_execz .LBB12_2
+; GFX1232-FAKE16-NEXT: s_and_saveexec_b32 s2, vcc_lo
+; GFX1232-FAKE16-NEXT: s_cbranch_execz .LBB12_4
; GFX1232-FAKE16-NEXT: ; %bb.1:
; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1232-FAKE16-NEXT: s_and_b32 s6, s2, 3
-; GFX1232-FAKE16-NEXT: s_and_b32 s7, s4, 0xff
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s6, s6, 3
-; GFX1232-FAKE16-NEXT: s_and_b32 s8, s2, -4
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s7, s7, s6
-; GFX1232-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v0, s7
-; GFX1232-FAKE16-NEXT: s_mov_b32 s10, -1
-; GFX1232-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX1232-FAKE16-NEXT: buffer_atomic_or_b32 v0, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1232-FAKE16-NEXT: s_and_b32 s4, s10, -4
+; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s11
+; GFX1232-FAKE16-NEXT: s_and_b32 s6, s10, 3
+; GFX1232-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s10, s6, 3
+; GFX1232-FAKE16-NEXT: s_and_b32 s6, s1, 0xff
+; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s0
+; GFX1232-FAKE16-NEXT: .LBB12_2: ; %atomicrmw.start
+; GFX1232-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-FAKE16-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1232-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1232-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1232-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s6, v0
-; GFX1232-FAKE16-NEXT: .LBB12_2:
-; GFX1232-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1232-FAKE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v1
+; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1232-FAKE16-NEXT: s_or_b32 s3, s0, s3
+; GFX1232-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1232-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
+; GFX1232-FAKE16-NEXT: s_cbranch_execnz .LBB12_2
+; GFX1232-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1232-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1232-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s10, v2
+; GFX1232-FAKE16-NEXT: .LBB12_4: ; %Flow
+; GFX1232-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s2
+; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1232-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1232-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1232-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1232-FAKE16-NEXT: v_cndmask_b32_e64 v0, s4, 0, vcc_lo
+; GFX1232-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1232-FAKE16-NEXT: s_mov_b32 s10, -1
+; GFX1232-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1232-FAKE16-NEXT: v_cndmask_b32_e64 v0, s1, 0, vcc_lo
+; GFX1232-FAKE16-NEXT: s_wait_alu 0xf1ff
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1232-FAKE16-NEXT: v_or_b32_e32 v0, s2, v0
-; GFX1232-FAKE16-NEXT: s_mov_b32 s2, -1
-; GFX1232-FAKE16-NEXT: buffer_store_b8 v0, off, s[0:3], null
+; GFX1232-FAKE16-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX1232-FAKE16-NEXT: buffer_store_b8 v0, off, s[8:11], null
; GFX1232-FAKE16-NEXT: s_endpgm
- %rmw = atomicrmw or ptr addrspace(1) %uniform.ptr, i8 %val monotonic, align 1, !amdgpu.no.fine.grained.memory !0
+ %rmw = atomicrmw or ptr addrspace(1) %uniform.ptr, i8 %val monotonic, align 1
store i8 %rmw, ptr addrspace(1) %result
ret void
}
@@ -8432,7 +9655,7 @@ define amdgpu_kernel void @uniform_add_i8(ptr addrspace(1) %result, ptr addrspac
; GFX1232-FAKE16-NEXT: s_mov_b32 s2, -1
; GFX1232-FAKE16-NEXT: buffer_store_b8 v0, off, s[0:3], null
; GFX1232-FAKE16-NEXT: s_endpgm
- %rmw = atomicrmw add ptr addrspace(1) %uniform.ptr, i8 %val monotonic, align 1, !amdgpu.no.fine.grained.memory !0
+ %rmw = atomicrmw add ptr addrspace(1) %uniform.ptr, i8 %val monotonic, align 1
store i8 %rmw, ptr addrspace(1) %result
ret void
}
@@ -8804,7 +10027,7 @@ define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspa
; GFX1232-NEXT: s_mov_b32 s2, -1
; GFX1232-NEXT: buffer_store_b8 v0, off, s[0:3], null
; GFX1232-NEXT: s_endpgm
- %rmw = atomicrmw xchg ptr addrspace(1) %uniform.ptr, i8 %val monotonic, align 1, !amdgpu.no.fine.grained.memory !0
+ %rmw = atomicrmw xchg ptr addrspace(1) %uniform.ptr, i8 %val monotonic, align 1
store i8 %rmw, ptr addrspace(1) %result
ret void
}
@@ -8812,500 +10035,712 @@ define amdgpu_kernel void @uniform_xchg_i8(ptr addrspace(1) %result, ptr addrspa
define amdgpu_kernel void @uniform_or_i16(ptr addrspace(1) %result, ptr addrspace(1) %uniform.ptr, i16 %val) {
; GFX7LESS-LABEL: uniform_or_i16:
; GFX7LESS: ; %bb.0:
-; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xd
+; GFX7LESS-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX7LESS-NEXT: s_load_dword s12, s[4:5], 0xd
; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, exec_lo, 0
; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, exec_hi, v0
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr0
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX7LESS-NEXT: s_cbranch_execz .LBB15_2
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX7LESS-NEXT: s_cbranch_execz .LBB15_4
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_and_b32 s8, s2, -4
+; GFX7LESS-NEXT: s_and_b32 s4, s10, -4
+; GFX7LESS-NEXT: s_mov_b32 s5, s11
+; GFX7LESS-NEXT: s_and_b32 s0, s10, 3
+; GFX7LESS-NEXT: s_and_b32 s1, s12, 0xffff
+; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0x0
+; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_lshl_b32 s13, s0, 3
+; GFX7LESS-NEXT: s_lshl_b32 s14, s1, s13
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: .LBB15_2: ; %atomicrmw.start
+; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX7LESS-NEXT: s_waitcnt expcnt(0)
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
+; GFX7LESS-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
+; GFX7LESS-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX7LESS-NEXT: s_cbranch_execnz .LBB15_2
+; GFX7LESS-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX7LESS-NEXT: v_bfe_u32 v0, v2, s13, 16
+; GFX7LESS-NEXT: .LBB15_4: ; %Flow
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_mov_b32 s11, 0xf000
-; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
-; GFX7LESS-NEXT: s_lshl_b32 s2, s2, 3
-; GFX7LESS-NEXT: s_and_b32 s7, s6, 0xffff
-; GFX7LESS-NEXT: s_lshl_b32 s7, s7, s2
; GFX7LESS-NEXT: s_mov_b32 s10, -1
-; GFX7LESS-NEXT: s_mov_b32 s9, s3
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s7
-; GFX7LESS-NEXT: buffer_atomic_or v0, off, s[8:11], 0 glc
-; GFX7LESS-NEXT: s_waitcnt vmcnt(0) expcnt(0)
-; GFX7LESS-NEXT: v_bfe_u32 v0, v0, s2, 16
-; GFX7LESS-NEXT: .LBB15_2:
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s6
-; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v0
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s12
+; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v0
; GFX7LESS-NEXT: v_cndmask_b32_e64 v0, v1, 0, vcc
-; GFX7LESS-NEXT: v_or_b32_e32 v0, s4, v0
-; GFX7LESS-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX7LESS-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX7LESS-NEXT: buffer_store_short v0, off, s[8:11], 0
; GFX7LESS-NEXT: s_endpgm
;
; GFX8-LABEL: uniform_or_i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX8-NEXT: s_load_dword s12, s[4:5], 0x34
; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr0
-; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX8-NEXT: s_cbranch_execz .LBB15_2
+; GFX8-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX8-NEXT: s_cbranch_execz .LBB15_4
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s8, s2, -4
-; GFX8-NEXT: s_and_b32 s2, s2, 3
-; GFX8-NEXT: s_mov_b32 s9, s3
-; GFX8-NEXT: s_lshl_b32 s2, s2, 3
-; GFX8-NEXT: s_and_b32 s3, 0xffff, s6
-; GFX8-NEXT: s_lshl_b32 s3, s3, s2
-; GFX8-NEXT: s_mov_b32 s11, 0xf000
-; GFX8-NEXT: s_mov_b32 s10, -1
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: buffer_atomic_or v0, off, s[8:11], 0 glc
+; GFX8-NEXT: s_and_b32 s4, s10, -4
+; GFX8-NEXT: s_mov_b32 s5, s11
+; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0
+; GFX8-NEXT: s_and_b32 s0, s10, 3
+; GFX8-NEXT: s_lshl_b32 s13, s0, 3
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s12
+; GFX8-NEXT: s_lshl_b32 s14, s0, s13
+; GFX8-NEXT: s_mov_b64 s[10:11], 0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: .LBB15_2: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX8-NEXT: v_mov_b32_e32 v3, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, s2, v0
-; GFX8-NEXT: .LBB15_2:
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
+; GFX8-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX8-NEXT: v_mov_b32_e32 v1, v2
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX8-NEXT: s_cbranch_execnz .LBB15_2
+; GFX8-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX8-NEXT: .LBB15_4: ; %Flow
+; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX8-NEXT: v_readfirstlane_b32 s4, v0
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: v_mov_b32_e32 v0, s12
; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
-; GFX8-NEXT: v_or_b32_e32 v0, s4, v0
-; GFX8-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX8-NEXT: s_mov_b32 s11, 0xf000
+; GFX8-NEXT: s_mov_b32 s10, -1
+; GFX8-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT: buffer_store_short v0, off, s[8:11], 0
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: uniform_or_i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s12, s[4:5], 0x34
; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr0
-; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX9-NEXT: s_cbranch_execz .LBB15_2
+; GFX9-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9-NEXT: s_cbranch_execz .LBB15_4
; GFX9-NEXT: ; %bb.1:
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s8, s2, -4
-; GFX9-NEXT: s_and_b32 s2, s2, 3
-; GFX9-NEXT: s_mov_b32 s9, s3
-; GFX9-NEXT: s_lshl_b32 s2, s2, 3
-; GFX9-NEXT: s_and_b32 s3, 0xffff, s6
-; GFX9-NEXT: s_lshl_b32 s3, s3, s2
-; GFX9-NEXT: s_mov_b32 s11, 0xf000
-; GFX9-NEXT: s_mov_b32 s10, -1
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: buffer_atomic_or v0, off, s[8:11], 0 glc
+; GFX9-NEXT: s_and_b32 s4, s10, -4
+; GFX9-NEXT: s_mov_b32 s5, s11
+; GFX9-NEXT: s_load_dword s1, s[4:5], 0x0
+; GFX9-NEXT: s_and_b32 s0, s10, 3
+; GFX9-NEXT: s_lshl_b32 s13, s0, 3
+; GFX9-NEXT: s_and_b32 s0, 0xffff, s12
+; GFX9-NEXT: s_lshl_b32 s14, s0, s13
+; GFX9-NEXT: s_mov_b64 s[10:11], 0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: .LBB15_2: ; %atomicrmw.start
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshrrev_b32_e32 v0, s2, v0
-; GFX9-NEXT: .LBB15_2:
-; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
+; GFX9-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX9-NEXT: v_mov_b32_e32 v1, v2
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX9-NEXT: s_cbranch_execnz .LBB15_2
+; GFX9-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX9-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX9-NEXT: .LBB15_4: ; %Flow
+; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_readfirstlane_b32 s4, v0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-NEXT: v_mov_b32_e32 v0, s12
; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
-; GFX9-NEXT: s_mov_b32 s3, 0xf000
-; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: v_or_b32_e32 v0, s4, v0
-; GFX9-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX9-NEXT: s_mov_b32 s11, 0xf000
+; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX9-NEXT: buffer_store_short v0, off, s[8:11], 0
; GFX9-NEXT: s_endpgm
;
; GFX1064-LABEL: uniform_or_i16:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_clause 0x1
-; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX1064-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX1064-NEXT: s_load_dword s12, s[4:5], 0x34
; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1064-NEXT: ; implicit-def: $vgpr0
-; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX1064-NEXT: s_cbranch_execz .LBB15_2
+; GFX1064-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064-NEXT: s_cbranch_execz .LBB15_4
; GFX1064-NEXT: ; %bb.1:
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_and_b32 s7, s2, 3
-; GFX1064-NEXT: s_and_b32 s8, 0xffff, s6
-; GFX1064-NEXT: s_lshl_b32 s7, s7, 3
-; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1064-NEXT: s_lshl_b32 s9, s8, s7
-; GFX1064-NEXT: s_and_b32 s8, s2, -4
-; GFX1064-NEXT: v_mov_b32_e32 v0, s9
-; GFX1064-NEXT: s_mov_b32 s10, -1
-; GFX1064-NEXT: s_mov_b32 s9, s3
-; GFX1064-NEXT: buffer_atomic_or v0, off, s[8:11], 0 glc
+; GFX1064-NEXT: s_and_b32 s4, s10, -4
+; GFX1064-NEXT: s_mov_b32 s5, s11
+; GFX1064-NEXT: s_and_b32 s1, s10, 3
+; GFX1064-NEXT: s_load_dword s0, s[4:5], 0x0
+; GFX1064-NEXT: s_lshl_b32 s13, s1, 3
+; GFX1064-NEXT: s_and_b32 s1, 0xffff, s12
+; GFX1064-NEXT: s_mov_b64 s[10:11], 0
+; GFX1064-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1064-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1064-NEXT: s_mov_b32 s6, -1
+; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064-NEXT: v_mov_b32_e32 v1, s0
+; GFX1064-NEXT: .LBB15_2: ; %atomicrmw.start
+; GFX1064-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1064-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1064-NEXT: v_mov_b32_e32 v3, v1
+; GFX1064-NEXT: v_mov_b32_e32 v2, v0
+; GFX1064-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX1064-NEXT: s_waitcnt vmcnt(0)
-; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s7, v0
-; GFX1064-NEXT: .LBB15_2:
-; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
+; GFX1064-NEXT: v_mov_b32_e32 v1, v2
+; GFX1064-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1064-NEXT: s_andn2_b64 exec, exec, s[10:11]
+; GFX1064-NEXT: s_cbranch_execnz .LBB15_2
+; GFX1064-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1064-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1064-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1064-NEXT: .LBB15_4: ; %Flow
+; GFX1064-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1064-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1064-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1064-NEXT: v_cndmask_b32_e64 v0, s6, 0, vcc
-; GFX1064-NEXT: v_or_b32_e32 v0, s2, v0
-; GFX1064-NEXT: s_mov_b32 s2, -1
-; GFX1064-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX1064-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1064-NEXT: s_mov_b32 s10, -1
+; GFX1064-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1064-NEXT: v_cndmask_b32_e64 v0, s12, 0, vcc
+; GFX1064-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX1064-NEXT: buffer_store_short v0, off, s[8:11], 0
; GFX1064-NEXT: s_endpgm
;
; GFX1032-LABEL: uniform_or_i16:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_clause 0x1
-; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX1032-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX1032-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX1032-NEXT: s_load_dword s1, s[4:5], 0x34
; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1032-NEXT: s_mov_b32 s3, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1032-NEXT: ; implicit-def: $vgpr0
-; GFX1032-NEXT: s_and_saveexec_b32 s4, vcc_lo
-; GFX1032-NEXT: s_cbranch_execz .LBB15_2
+; GFX1032-NEXT: s_and_saveexec_b32 s2, vcc_lo
+; GFX1032-NEXT: s_cbranch_execz .LBB15_4
; GFX1032-NEXT: ; %bb.1:
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_and_b32 s5, s2, 3
-; GFX1032-NEXT: s_and_b32 s7, 0xffff, s6
-; GFX1032-NEXT: s_lshl_b32 s5, s5, 3
-; GFX1032-NEXT: s_and_b32 s8, s2, -4
-; GFX1032-NEXT: s_lshl_b32 s7, s7, s5
-; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1032-NEXT: v_mov_b32_e32 v0, s7
-; GFX1032-NEXT: s_mov_b32 s10, -1
-; GFX1032-NEXT: s_mov_b32 s9, s3
-; GFX1032-NEXT: buffer_atomic_or v0, off, s[8:11], 0 glc
+; GFX1032-NEXT: s_and_b32 s4, s10, -4
+; GFX1032-NEXT: s_mov_b32 s5, s11
+; GFX1032-NEXT: s_and_b32 s6, s10, 3
+; GFX1032-NEXT: s_load_dword s0, s[4:5], 0x0
+; GFX1032-NEXT: s_lshl_b32 s10, s6, 3
+; GFX1032-NEXT: s_and_b32 s6, 0xffff, s1
+; GFX1032-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1032-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1032-NEXT: s_mov_b32 s6, -1
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: v_mov_b32_e32 v1, s0
+; GFX1032-NEXT: .LBB15_2: ; %atomicrmw.start
+; GFX1032-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1032-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1032-NEXT: v_mov_b32_e32 v3, v1
+; GFX1032-NEXT: v_mov_b32_e32 v2, v0
+; GFX1032-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; GFX1032-NEXT: s_waitcnt vmcnt(0)
-; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s5, v0
-; GFX1032-NEXT: .LBB15_2:
-; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX1032-NEXT: v_cmp_eq_u32_e64 s0, v2, v1
+; GFX1032-NEXT: v_mov_b32_e32 v1, v2
+; GFX1032-NEXT: s_or_b32 s3, s0, s3
+; GFX1032-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
+; GFX1032-NEXT: s_cbranch_execnz .LBB15_2
+; GFX1032-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1032-NEXT: v_lshrrev_b32_e32 v0, s10, v2
+; GFX1032-NEXT: .LBB15_4: ; %Flow
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1032-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1032-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1032-NEXT: v_cndmask_b32_e64 v0, s6, 0, vcc_lo
-; GFX1032-NEXT: v_or_b32_e32 v0, s2, v0
-; GFX1032-NEXT: s_mov_b32 s2, -1
-; GFX1032-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1032-NEXT: s_mov_b32 s10, -1
+; GFX1032-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1032-NEXT: v_cndmask_b32_e64 v0, s1, 0, vcc_lo
+; GFX1032-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX1032-NEXT: buffer_store_short v0, off, s[8:11], 0
; GFX1032-NEXT: s_endpgm
;
; GFX1164-TRUE16-LABEL: uniform_or_i16:
; GFX1164-TRUE16: ; %bb.0:
; GFX1164-TRUE16-NEXT: s_clause 0x1
-; GFX1164-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1164-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
+; GFX1164-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1164-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1164-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX1164-TRUE16-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX1164-TRUE16-NEXT: s_cbranch_execz .LBB15_2
+; GFX1164-TRUE16-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1164-TRUE16-NEXT: s_cbranch_execz .LBB15_4
; GFX1164-TRUE16-NEXT: ; %bb.1:
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-TRUE16-NEXT: s_and_b32 s7, s2, 3
-; GFX1164-TRUE16-NEXT: s_and_b32 s8, 0xffff, s6
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s7, s7, 3
-; GFX1164-TRUE16-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1164-TRUE16-NEXT: s_lshl_b32 s9, s8, s7
-; GFX1164-TRUE16-NEXT: s_and_b32 s8, s2, -4
-; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v0, s9
-; GFX1164-TRUE16-NEXT: s_mov_b32 s10, -1
-; GFX1164-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX1164-TRUE16-NEXT: buffer_atomic_or_b32 v0, off, s[8:11], 0 glc
+; GFX1164-TRUE16-NEXT: s_and_b32 s4, s10, -4
+; GFX1164-TRUE16-NEXT: s_mov_b32 s5, s11
+; GFX1164-TRUE16-NEXT: s_and_b32 s1, s10, 3
+; GFX1164-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s13, s1, 3
+; GFX1164-TRUE16-NEXT: s_and_b32 s1, 0xffff, s12
+; GFX1164-TRUE16-NEXT: s_mov_b64 s[10:11], 0
+; GFX1164-TRUE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1164-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, s0
+; GFX1164-TRUE16-NEXT: .LBB15_2: ; %atomicrmw.start
+; GFX1164-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-TRUE16-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX1164-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s7, v0
-; GFX1164-TRUE16-NEXT: .LBB15_2:
-; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX1164-TRUE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
+; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1164-TRUE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1164-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1164-TRUE16-NEXT: s_cbranch_execnz .LBB15_2
+; GFX1164-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1164-TRUE16-NEXT: .LBB15_4: ; %Flow
+; GFX1164-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1164-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX1164-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1164-TRUE16-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1164-TRUE16-NEXT: s_mov_b32 s10, -1
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1164-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1164-TRUE16-NEXT: v_cndmask_b16 v0.l, s6, 0, vcc
-; GFX1164-TRUE16-NEXT: v_or_b16 v0.l, s2, v0.l
-; GFX1164-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX1164-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GFX1164-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-TRUE16-NEXT: v_cndmask_b16 v0.l, s12, 0, vcc
+; GFX1164-TRUE16-NEXT: v_or_b16 v0.l, s0, v0.l
+; GFX1164-TRUE16-NEXT: buffer_store_b16 v0, off, s[8:11], 0
; GFX1164-TRUE16-NEXT: s_endpgm
;
; GFX1164-FAKE16-LABEL: uniform_or_i16:
; GFX1164-FAKE16: ; %bb.0:
; GFX1164-FAKE16-NEXT: s_clause 0x1
-; GFX1164-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1164-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1164-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
+; GFX1164-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1164-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1164-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX1164-FAKE16-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX1164-FAKE16-NEXT: s_cbranch_execz .LBB15_2
+; GFX1164-FAKE16-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1164-FAKE16-NEXT: s_cbranch_execz .LBB15_4
; GFX1164-FAKE16-NEXT: ; %bb.1:
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-FAKE16-NEXT: s_and_b32 s7, s2, 3
-; GFX1164-FAKE16-NEXT: s_and_b32 s8, 0xffff, s6
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s7, s7, 3
-; GFX1164-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1164-FAKE16-NEXT: s_lshl_b32 s9, s8, s7
-; GFX1164-FAKE16-NEXT: s_and_b32 s8, s2, -4
-; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v0, s9
-; GFX1164-FAKE16-NEXT: s_mov_b32 s10, -1
-; GFX1164-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX1164-FAKE16-NEXT: buffer_atomic_or_b32 v0, off, s[8:11], 0 glc
+; GFX1164-FAKE16-NEXT: s_and_b32 s4, s10, -4
+; GFX1164-FAKE16-NEXT: s_mov_b32 s5, s11
+; GFX1164-FAKE16-NEXT: s_and_b32 s1, s10, 3
+; GFX1164-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s13, s1, 3
+; GFX1164-FAKE16-NEXT: s_and_b32 s1, 0xffff, s12
+; GFX1164-FAKE16-NEXT: s_mov_b64 s[10:11], 0
+; GFX1164-FAKE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1164-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1164-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, s0
+; GFX1164-FAKE16-NEXT: .LBB15_2: ; %atomicrmw.start
+; GFX1164-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-FAKE16-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX1164-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s7, v0
-; GFX1164-FAKE16-NEXT: .LBB15_2:
-; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1164-FAKE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
+; GFX1164-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1164-FAKE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1164-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1164-FAKE16-NEXT: s_cbranch_execnz .LBB15_2
+; GFX1164-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1164-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1164-FAKE16-NEXT: .LBB15_4: ; %Flow
+; GFX1164-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1164-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1164-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1164-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1164-FAKE16-NEXT: v_cndmask_b32_e64 v0, s6, 0, vcc
+; GFX1164-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1164-FAKE16-NEXT: s_mov_b32 s10, -1
+; GFX1164-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1164-FAKE16-NEXT: v_cndmask_b32_e64 v0, s12, 0, vcc
; GFX1164-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-FAKE16-NEXT: v_or_b32_e32 v0, s2, v0
-; GFX1164-FAKE16-NEXT: s_mov_b32 s2, -1
-; GFX1164-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GFX1164-FAKE16-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX1164-FAKE16-NEXT: buffer_store_b16 v0, off, s[8:11], 0
; GFX1164-FAKE16-NEXT: s_endpgm
;
; GFX1132-TRUE16-LABEL: uniform_or_i16:
; GFX1132-TRUE16: ; %bb.0:
; GFX1132-TRUE16-NEXT: s_clause 0x1
-; GFX1132-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132-TRUE16-NEXT: s_load_b32 s4, s[4:5], 0x34
+; GFX1132-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
+; GFX1132-TRUE16-NEXT: s_load_b32 s1, s[4:5], 0x34
; GFX1132-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-TRUE16-NEXT: s_mov_b32 s3, 0
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1132-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX1132-TRUE16-NEXT: s_and_saveexec_b32 s5, vcc_lo
-; GFX1132-TRUE16-NEXT: s_cbranch_execz .LBB15_2
+; GFX1132-TRUE16-NEXT: s_and_saveexec_b32 s2, vcc_lo
+; GFX1132-TRUE16-NEXT: s_cbranch_execz .LBB15_4
; GFX1132-TRUE16-NEXT: ; %bb.1:
; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-TRUE16-NEXT: s_and_b32 s6, s2, 3
-; GFX1132-TRUE16-NEXT: s_and_b32 s7, 0xffff, s4
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s6, s6, 3
-; GFX1132-TRUE16-NEXT: s_and_b32 s8, s2, -4
-; GFX1132-TRUE16-NEXT: s_lshl_b32 s7, s7, s6
-; GFX1132-TRUE16-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v0, s7
-; GFX1132-TRUE16-NEXT: s_mov_b32 s10, -1
-; GFX1132-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX1132-TRUE16-NEXT: buffer_atomic_or_b32 v0, off, s[8:11], 0 glc
+; GFX1132-TRUE16-NEXT: s_and_b32 s4, s10, -4
+; GFX1132-TRUE16-NEXT: s_mov_b32 s5, s11
+; GFX1132-TRUE16-NEXT: s_and_b32 s6, s10, 3
+; GFX1132-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s10, s6, 3
+; GFX1132-TRUE16-NEXT: s_and_b32 s6, 0xffff, s1
+; GFX1132-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-TRUE16-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, s0
+; GFX1132-TRUE16-NEXT: .LBB15_2: ; %atomicrmw.start
+; GFX1132-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-TRUE16-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1132-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s6, v0
-; GFX1132-TRUE16-NEXT: .LBB15_2:
-; GFX1132-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX1132-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v1
+; GFX1132-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1132-TRUE16-NEXT: s_or_b32 s3, s0, s3
+; GFX1132-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
+; GFX1132-TRUE16-NEXT: s_cbranch_execnz .LBB15_2
+; GFX1132-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1132-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1132-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s10, v2
+; GFX1132-TRUE16-NEXT: .LBB15_4: ; %Flow
+; GFX1132-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1132-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX1132-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1132-TRUE16-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1132-TRUE16-NEXT: s_mov_b32 s10, -1
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1132-TRUE16-NEXT: v_cndmask_b16 v0.l, s4, 0, vcc_lo
-; GFX1132-TRUE16-NEXT: v_or_b16 v0.l, s2, v0.l
-; GFX1132-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX1132-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GFX1132-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-TRUE16-NEXT: v_cndmask_b16 v0.l, s1, 0, vcc_lo
+; GFX1132-TRUE16-NEXT: v_or_b16 v0.l, s0, v0.l
+; GFX1132-TRUE16-NEXT: buffer_store_b16 v0, off, s[8:11], 0
; GFX1132-TRUE16-NEXT: s_endpgm
;
; GFX1132-FAKE16-LABEL: uniform_or_i16:
; GFX1132-FAKE16: ; %bb.0:
; GFX1132-FAKE16-NEXT: s_clause 0x1
-; GFX1132-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1132-FAKE16-NEXT: s_load_b32 s4, s[4:5], 0x34
+; GFX1132-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
+; GFX1132-FAKE16-NEXT: s_load_b32 s1, s[4:5], 0x34
; GFX1132-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1132-FAKE16-NEXT: s_mov_b32 s3, 0
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1132-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX1132-FAKE16-NEXT: s_and_saveexec_b32 s5, vcc_lo
-; GFX1132-FAKE16-NEXT: s_cbranch_execz .LBB15_2
+; GFX1132-FAKE16-NEXT: s_and_saveexec_b32 s2, vcc_lo
+; GFX1132-FAKE16-NEXT: s_cbranch_execz .LBB15_4
; GFX1132-FAKE16-NEXT: ; %bb.1:
; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-FAKE16-NEXT: s_and_b32 s6, s2, 3
-; GFX1132-FAKE16-NEXT: s_and_b32 s7, 0xffff, s4
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s6, s6, 3
-; GFX1132-FAKE16-NEXT: s_and_b32 s8, s2, -4
-; GFX1132-FAKE16-NEXT: s_lshl_b32 s7, s7, s6
-; GFX1132-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v0, s7
-; GFX1132-FAKE16-NEXT: s_mov_b32 s10, -1
-; GFX1132-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX1132-FAKE16-NEXT: buffer_atomic_or_b32 v0, off, s[8:11], 0 glc
+; GFX1132-FAKE16-NEXT: s_and_b32 s4, s10, -4
+; GFX1132-FAKE16-NEXT: s_mov_b32 s5, s11
+; GFX1132-FAKE16-NEXT: s_and_b32 s6, s10, 3
+; GFX1132-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s10, s6, 3
+; GFX1132-FAKE16-NEXT: s_and_b32 s6, 0xffff, s1
+; GFX1132-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1132-FAKE16-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1132-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, s0
+; GFX1132-FAKE16-NEXT: .LBB15_2: ; %atomicrmw.start
+; GFX1132-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-FAKE16-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1132-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1132-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s6, v0
-; GFX1132-FAKE16-NEXT: .LBB15_2:
-; GFX1132-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1132-FAKE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v1
+; GFX1132-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1132-FAKE16-NEXT: s_or_b32 s3, s0, s3
+; GFX1132-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
+; GFX1132-FAKE16-NEXT: s_cbranch_execnz .LBB15_2
+; GFX1132-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1132-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1132-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s10, v2
+; GFX1132-FAKE16-NEXT: .LBB15_4: ; %Flow
+; GFX1132-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s2
+; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1132-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1132-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1132-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1132-FAKE16-NEXT: v_cndmask_b32_e64 v0, s4, 0, vcc_lo
+; GFX1132-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1132-FAKE16-NEXT: s_mov_b32 s10, -1
+; GFX1132-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1132-FAKE16-NEXT: v_cndmask_b32_e64 v0, s1, 0, vcc_lo
; GFX1132-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-FAKE16-NEXT: v_or_b32_e32 v0, s2, v0
-; GFX1132-FAKE16-NEXT: s_mov_b32 s2, -1
-; GFX1132-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0
+; GFX1132-FAKE16-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX1132-FAKE16-NEXT: buffer_store_b16 v0, off, s[8:11], 0
; GFX1132-FAKE16-NEXT: s_endpgm
;
; GFX1264-TRUE16-LABEL: uniform_or_i16:
; GFX1264-TRUE16: ; %bb.0:
; GFX1264-TRUE16-NEXT: s_clause 0x1
-; GFX1264-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1264-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
+; GFX1264-TRUE16-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1264-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1264-TRUE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1264-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX1264-TRUE16-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX1264-TRUE16-NEXT: s_cbranch_execz .LBB15_2
+; GFX1264-TRUE16-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1264-TRUE16-NEXT: s_cbranch_execz .LBB15_4
; GFX1264-TRUE16-NEXT: ; %bb.1:
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-TRUE16-NEXT: s_and_b32 s7, s2, 3
-; GFX1264-TRUE16-NEXT: s_and_b32 s8, 0xffff, s6
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s7, s7, 3
-; GFX1264-TRUE16-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1264-TRUE16-NEXT: s_lshl_b32 s9, s8, s7
-; GFX1264-TRUE16-NEXT: s_and_b32 s8, s2, -4
-; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v0, s9
-; GFX1264-TRUE16-NEXT: s_mov_b32 s10, -1
-; GFX1264-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX1264-TRUE16-NEXT: buffer_atomic_or_b32 v0, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1264-TRUE16-NEXT: s_and_b32 s4, s10, -4
+; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s11
+; GFX1264-TRUE16-NEXT: s_and_b32 s1, s10, 3
+; GFX1264-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s13, s1, 3
+; GFX1264-TRUE16-NEXT: s_and_b32 s1, 0xffff, s12
+; GFX1264-TRUE16-NEXT: s_mov_b64 s[10:11], 0
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s0
+; GFX1264-TRUE16-NEXT: .LBB15_2: ; %atomicrmw.start
+; GFX1264-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1264-TRUE16-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX1264-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s7, v0
-; GFX1264-TRUE16-NEXT: .LBB15_2:
-; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
+; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1264-TRUE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1264-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1264-TRUE16-NEXT: s_cbranch_execnz .LBB15_2
+; GFX1264-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1264-TRUE16-NEXT: .LBB15_4: ; %Flow
+; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1264-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1264-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1264-TRUE16-NEXT: v_cndmask_b16 v0.l, s6, 0, vcc
-; GFX1264-TRUE16-NEXT: v_or_b16 v0.l, s2, v0.l
-; GFX1264-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX1264-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX1264-TRUE16-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1264-TRUE16-NEXT: s_mov_b32 s10, -1
+; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1264-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1264-TRUE16-NEXT: v_cndmask_b16 v0.l, s12, 0, vcc
+; GFX1264-TRUE16-NEXT: s_wait_alu 0xf1ff
+; GFX1264-TRUE16-NEXT: v_or_b16 v0.l, s0, v0.l
+; GFX1264-TRUE16-NEXT: buffer_store_b16 v0, off, s[8:11], null
; GFX1264-TRUE16-NEXT: s_endpgm
;
; GFX1264-FAKE16-LABEL: uniform_or_i16:
; GFX1264-FAKE16: ; %bb.0:
; GFX1264-FAKE16-NEXT: s_clause 0x1
-; GFX1264-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX1264-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
+; GFX1264-FAKE16-NEXT: s_load_b32 s12, s[4:5], 0x34
; GFX1264-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1264-FAKE16-NEXT: v_mbcnt_hi_u32_b32 v0, exec_hi, v0
; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX1264-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX1264-FAKE16-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GFX1264-FAKE16-NEXT: s_cbranch_execz .LBB15_2
+; GFX1264-FAKE16-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1264-FAKE16-NEXT: s_cbranch_execz .LBB15_4
; GFX1264-FAKE16-NEXT: ; %bb.1:
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-FAKE16-NEXT: s_and_b32 s7, s2, 3
-; GFX1264-FAKE16-NEXT: s_and_b32 s8, 0xffff, s6
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s7, s7, 3
-; GFX1264-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1264-FAKE16-NEXT: s_lshl_b32 s9, s8, s7
-; GFX1264-FAKE16-NEXT: s_and_b32 s8, s2, -4
-; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v0, s9
-; GFX1264-FAKE16-NEXT: s_mov_b32 s10, -1
-; GFX1264-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX1264-FAKE16-NEXT: buffer_atomic_or_b32 v0, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1264-FAKE16-NEXT: s_and_b32 s4, s10, -4
+; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s11
+; GFX1264-FAKE16-NEXT: s_and_b32 s1, s10, 3
+; GFX1264-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s13, s1, 3
+; GFX1264-FAKE16-NEXT: s_and_b32 s1, 0xffff, s12
+; GFX1264-FAKE16-NEXT: s_mov_b64 s[10:11], 0
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s14, s1, s13
+; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s0
+; GFX1264-FAKE16-NEXT: .LBB15_2: ; %atomicrmw.start
+; GFX1264-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1264-FAKE16-NEXT: v_or_b32_e32 v0, s14, v1
+; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX1264-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s7, v0
-; GFX1264-FAKE16-NEXT: .LBB15_2:
-; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v1
+; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1264-FAKE16-NEXT: s_or_b64 s[10:11], s[0:1], s[10:11]
+; GFX1264-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[10:11]
+; GFX1264-FAKE16-NEXT: s_cbranch_execnz .LBB15_2
+; GFX1264-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX1264-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s13, v2
+; GFX1264-FAKE16-NEXT: .LBB15_4: ; %Flow
+; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1264-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1264-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1264-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1264-FAKE16-NEXT: v_cndmask_b32_e64 v0, s6, 0, vcc
+; GFX1264-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1264-FAKE16-NEXT: s_mov_b32 s10, -1
+; GFX1264-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1264-FAKE16-NEXT: v_cndmask_b32_e64 v0, s12, 0, vcc
+; GFX1264-FAKE16-NEXT: s_wait_alu 0xf1ff
; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1264-FAKE16-NEXT: v_or_b32_e32 v0, s2, v0
-; GFX1264-FAKE16-NEXT: s_mov_b32 s2, -1
-; GFX1264-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX1264-FAKE16-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX1264-FAKE16-NEXT: buffer_store_b16 v0, off, s[8:11], null
; GFX1264-FAKE16-NEXT: s_endpgm
;
; GFX1232-TRUE16-LABEL: uniform_or_i16:
; GFX1232-TRUE16: ; %bb.0:
; GFX1232-TRUE16-NEXT: s_clause 0x1
-; GFX1232-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1232-TRUE16-NEXT: s_load_b32 s4, s[4:5], 0x34
+; GFX1232-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
+; GFX1232-TRUE16-NEXT: s_load_b32 s1, s[4:5], 0x34
; GFX1232-TRUE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1232-TRUE16-NEXT: s_mov_b32 s3, 0
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1232-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1232-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX1232-TRUE16-NEXT: s_and_saveexec_b32 s5, vcc_lo
-; GFX1232-TRUE16-NEXT: s_cbranch_execz .LBB15_2
+; GFX1232-TRUE16-NEXT: s_and_saveexec_b32 s2, vcc_lo
+; GFX1232-TRUE16-NEXT: s_cbranch_execz .LBB15_4
; GFX1232-TRUE16-NEXT: ; %bb.1:
; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1232-TRUE16-NEXT: s_and_b32 s6, s2, 3
-; GFX1232-TRUE16-NEXT: s_and_b32 s7, 0xffff, s4
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s6, s6, 3
-; GFX1232-TRUE16-NEXT: s_and_b32 s8, s2, -4
-; GFX1232-TRUE16-NEXT: s_lshl_b32 s7, s7, s6
-; GFX1232-TRUE16-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v0, s7
-; GFX1232-TRUE16-NEXT: s_mov_b32 s10, -1
-; GFX1232-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX1232-TRUE16-NEXT: buffer_atomic_or_b32 v0, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1232-TRUE16-NEXT: s_and_b32 s4, s10, -4
+; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s11
+; GFX1232-TRUE16-NEXT: s_and_b32 s6, s10, 3
+; GFX1232-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s10, s6, 3
+; GFX1232-TRUE16-NEXT: s_and_b32 s6, 0xffff, s1
+; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s0
+; GFX1232-TRUE16-NEXT: .LBB15_2: ; %atomicrmw.start
+; GFX1232-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-TRUE16-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1232-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1232-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1232-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s6, v0
-; GFX1232-TRUE16-NEXT: .LBB15_2:
-; GFX1232-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX1232-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v1
+; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1232-TRUE16-NEXT: s_or_b32 s3, s0, s3
+; GFX1232-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1232-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
+; GFX1232-TRUE16-NEXT: s_cbranch_execnz .LBB15_2
+; GFX1232-TRUE16-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1232-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1232-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s10, v2
+; GFX1232-TRUE16-NEXT: .LBB15_4: ; %Flow
+; GFX1232-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1232-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1232-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1232-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1232-TRUE16-NEXT: v_cndmask_b16 v0.l, s4, 0, vcc_lo
-; GFX1232-TRUE16-NEXT: v_or_b16 v0.l, s2, v0.l
-; GFX1232-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX1232-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX1232-TRUE16-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1232-TRUE16-NEXT: s_mov_b32 s10, -1
+; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1232-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1232-TRUE16-NEXT: v_cndmask_b16 v0.l, s1, 0, vcc_lo
+; GFX1232-TRUE16-NEXT: s_wait_alu 0xf1ff
+; GFX1232-TRUE16-NEXT: v_or_b16 v0.l, s0, v0.l
+; GFX1232-TRUE16-NEXT: buffer_store_b16 v0, off, s[8:11], null
; GFX1232-TRUE16-NEXT: s_endpgm
;
; GFX1232-FAKE16-LABEL: uniform_or_i16:
; GFX1232-FAKE16: ; %bb.0:
; GFX1232-FAKE16-NEXT: s_clause 0x1
-; GFX1232-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1232-FAKE16-NEXT: s_load_b32 s4, s[4:5], 0x34
+; GFX1232-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
+; GFX1232-FAKE16-NEXT: s_load_b32 s1, s[4:5], 0x34
; GFX1232-FAKE16-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
+; GFX1232-FAKE16-NEXT: s_mov_b32 s3, 0
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1232-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX1232-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX1232-FAKE16-NEXT: s_and_saveexec_b32 s5, vcc_lo
-; GFX1232-FAKE16-NEXT: s_cbranch_execz .LBB15_2
+; GFX1232-FAKE16-NEXT: s_and_saveexec_b32 s2, vcc_lo
+; GFX1232-FAKE16-NEXT: s_cbranch_execz .LBB15_4
; GFX1232-FAKE16-NEXT: ; %bb.1:
; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1232-FAKE16-NEXT: s_and_b32 s6, s2, 3
-; GFX1232-FAKE16-NEXT: s_and_b32 s7, 0xffff, s4
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s6, s6, 3
-; GFX1232-FAKE16-NEXT: s_and_b32 s8, s2, -4
-; GFX1232-FAKE16-NEXT: s_lshl_b32 s7, s7, s6
-; GFX1232-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v0, s7
-; GFX1232-FAKE16-NEXT: s_mov_b32 s10, -1
-; GFX1232-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX1232-FAKE16-NEXT: buffer_atomic_or_b32 v0, off, s[8:11], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1232-FAKE16-NEXT: s_and_b32 s4, s10, -4
+; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s11
+; GFX1232-FAKE16-NEXT: s_and_b32 s6, s10, 3
+; GFX1232-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x0
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s10, s6, 3
+; GFX1232-FAKE16-NEXT: s_and_b32 s6, 0xffff, s1
+; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s11, s6, s10
+; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s0
+; GFX1232-FAKE16-NEXT: .LBB15_2: ; %atomicrmw.start
+; GFX1232-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-FAKE16-NEXT: v_or_b32_e32 v0, s11, v1
+; GFX1232-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1232-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1232-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s6, v0
-; GFX1232-FAKE16-NEXT: .LBB15_2:
-; GFX1232-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1232-FAKE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v1
+; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1232-FAKE16-NEXT: s_or_b32 s3, s0, s3
+; GFX1232-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1232-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
+; GFX1232-FAKE16-NEXT: s_cbranch_execnz .LBB15_2
+; GFX1232-FAKE16-NEXT: ; %bb.3: ; %atomicrmw.end
+; GFX1232-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1232-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s10, v2
+; GFX1232-FAKE16-NEXT: .LBB15_4: ; %Flow
+; GFX1232-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s2
+; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX1232-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1232-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1232-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1232-FAKE16-NEXT: v_cndmask_b32_e64 v0, s4, 0, vcc_lo
+; GFX1232-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1232-FAKE16-NEXT: s_mov_b32 s10, -1
+; GFX1232-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1232-FAKE16-NEXT: v_cndmask_b32_e64 v0, s1, 0, vcc_lo
+; GFX1232-FAKE16-NEXT: s_wait_alu 0xf1ff
; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1232-FAKE16-NEXT: v_or_b32_e32 v0, s2, v0
-; GFX1232-FAKE16-NEXT: s_mov_b32 s2, -1
-; GFX1232-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
+; GFX1232-FAKE16-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX1232-FAKE16-NEXT: buffer_store_b16 v0, off, s[8:11], null
; GFX1232-FAKE16-NEXT: s_endpgm
- %rmw = atomicrmw or ptr addrspace(1) %uniform.ptr, i16 %val monotonic, align 2, !amdgpu.no.fine.grained.memory !0
+ %rmw = atomicrmw or ptr addrspace(1) %uniform.ptr, i16 %val monotonic, align 2
store i16 %rmw, ptr addrspace(1) %result
ret void
}
@@ -10117,7 +11552,7 @@ define amdgpu_kernel void @uniform_add_i16(ptr addrspace(1) %result, ptr addrspa
; GFX1232-FAKE16-NEXT: s_mov_b32 s2, -1
; GFX1232-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1232-FAKE16-NEXT: s_endpgm
- %rmw = atomicrmw add ptr addrspace(1) %uniform.ptr, i16 %val monotonic, align 2, !amdgpu.no.fine.grained.memory !0
+ %rmw = atomicrmw add ptr addrspace(1) %uniform.ptr, i16 %val monotonic, align 2
store i16 %rmw, ptr addrspace(1) %result
ret void
}
@@ -10489,7 +11924,7 @@ define amdgpu_kernel void @uniform_xchg_i16(ptr addrspace(1) %result, ptr addrsp
; GFX1232-NEXT: s_mov_b32 s2, -1
; GFX1232-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1232-NEXT: s_endpgm
- %rmw = atomicrmw xchg ptr addrspace(1) %uniform.ptr, i16 %val monotonic, align 2, !amdgpu.no.fine.grained.memory !0
+ %rmw = atomicrmw xchg ptr addrspace(1) %uniform.ptr, i16 %val monotonic, align 2
store i16 %rmw, ptr addrspace(1) %result
ret void
}
@@ -11076,7 +12511,7 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1232-FAKE16-NEXT: s_mov_b32 s2, -1
; GFX1232-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1232-FAKE16-NEXT: s_endpgm
- %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, half %val monotonic, align 2, !amdgpu.no.fine.grained.memory !0
+ %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, half %val monotonic, align 2
store half %rmw, ptr addrspace(1) %result
ret void
}
@@ -11766,7 +13201,7 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1232-FAKE16-NEXT: s_mov_b32 s2, -1
; GFX1232-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1232-FAKE16-NEXT: s_endpgm
- %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, bfloat %val monotonic, align 2, !amdgpu.no.fine.grained.memory !0
+ %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, bfloat %val monotonic, align 2
store bfloat %rmw, ptr addrspace(1) %result
ret void
}
@@ -12037,32 +13472,72 @@ define amdgpu_kernel void @uniform_fadd_v2f16(ptr addrspace(1) %result, ptr addr
; GFX1264-LABEL: uniform_fadd_v2f16:
; GFX1264: ; %bb.0:
; GFX1264-NEXT: s_clause 0x1
-; GFX1264-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-NEXT: v_mov_b32_e32 v0, 0
+; GFX1264-NEXT: s_load_b32 s10, s[4:5], 0x34
+; GFX1264-NEXT: s_mov_b64 s[8:9], 0
+; GFX1264-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1264-NEXT: s_mov_b32 s6, -1
; GFX1264-NEXT: s_wait_kmcnt 0x0
-; GFX1264-NEXT: v_mov_b32_e32 v1, s6
-; GFX1264-NEXT: global_atomic_pk_add_f16 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1264-NEXT: s_load_b32 s4, s[2:3], 0x0
+; GFX1264-NEXT: s_mov_b32 s5, s3
+; GFX1264-NEXT: s_wait_kmcnt 0x0
+; GFX1264-NEXT: v_mov_b32_e32 v1, s4
+; GFX1264-NEXT: s_mov_b32 s4, s2
+; GFX1264-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX1264-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1264-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1264-NEXT: v_pk_add_f16 v0, v1, s10
+; GFX1264-NEXT: v_mov_b32_e32 v3, v1
+; GFX1264-NEXT: v_mov_b32_e32 v2, v0
+; GFX1264-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1264-NEXT: s_wait_loadcnt 0x0
+; GFX1264-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1264-NEXT: v_mov_b32_e32 v1, v2
+; GFX1264-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX1264-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-NEXT: s_and_not1_b64 exec, exec, s[8:9]
+; GFX1264-NEXT: s_cbranch_execnz .LBB20_1
+; GFX1264-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1264-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1264-NEXT: s_mov_b32 s3, 0x31016000
; GFX1264-NEXT: s_mov_b32 s2, -1
-; GFX1264-NEXT: s_wait_loadcnt 0x0
-; GFX1264-NEXT: buffer_store_b32 v0, off, s[0:3], null
+; GFX1264-NEXT: buffer_store_b32 v2, off, s[0:3], null
; GFX1264-NEXT: s_endpgm
;
; GFX1232-LABEL: uniform_fadd_v2f16:
; GFX1232: ; %bb.0:
; GFX1232-NEXT: s_clause 0x1
-; GFX1232-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1232-NEXT: s_load_b32 s8, s[4:5], 0x34
+; GFX1232-NEXT: s_mov_b32 s9, 0
+; GFX1232-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-NEXT: s_mov_b32 s6, -1
+; GFX1232-NEXT: s_wait_kmcnt 0x0
+; GFX1232-NEXT: s_load_b32 s4, s[2:3], 0x0
+; GFX1232-NEXT: s_mov_b32 s5, s3
; GFX1232-NEXT: s_wait_kmcnt 0x0
-; GFX1232-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
-; GFX1232-NEXT: global_atomic_pk_add_f16 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1232-NEXT: v_mov_b32_e32 v1, s4
+; GFX1232-NEXT: s_mov_b32 s4, s2
+; GFX1232-NEXT: .LBB20_1: ; %atomicrmw.start
+; GFX1232-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1232-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-NEXT: v_pk_add_f16 v0, v1, s8
+; GFX1232-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1232-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1232-NEXT: s_wait_loadcnt 0x0
+; GFX1232-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1232-NEXT: v_mov_b32_e32 v1, v2
+; GFX1232-NEXT: s_or_b32 s9, vcc_lo, s9
+; GFX1232-NEXT: s_wait_alu 0xfffe
+; GFX1232-NEXT: s_and_not1_b32 exec_lo, exec_lo, s9
+; GFX1232-NEXT: s_cbranch_execnz .LBB20_1
+; GFX1232-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1232-NEXT: s_or_b32 exec_lo, exec_lo, s9
; GFX1232-NEXT: s_mov_b32 s3, 0x31016000
; GFX1232-NEXT: s_mov_b32 s2, -1
-; GFX1232-NEXT: s_wait_loadcnt 0x0
-; GFX1232-NEXT: buffer_store_b32 v0, off, s[0:3], null
+; GFX1232-NEXT: buffer_store_b32 v2, off, s[0:3], null
; GFX1232-NEXT: s_endpgm
- %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, <2 x half> %val monotonic, align 4, !amdgpu.no.fine.grained.memory !0
+ %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, <2 x half> %val monotonic, align 4
store <2 x half> %rmw, ptr addrspace(1) %result
ret void
}
@@ -12357,15 +13832,13 @@ define amdgpu_kernel void @uniform_fadd_v2bf16(ptr addrspace(1) %result, ptr add
; GFX1164-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
; GFX1164-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
; GFX1164-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
-; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1164-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc
+; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1164-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
; GFX1164-TRUE16-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX1164-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc
-; GFX1164-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2
+; GFX1164-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc
+; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1164-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -12475,14 +13948,12 @@ define amdgpu_kernel void @uniform_fadd_v2bf16(ptr addrspace(1) %result, ptr add
; GFX1132-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
; GFX1132-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
; GFX1132-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
-; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1132-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1132-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc_lo
; GFX1132-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX1132-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
-; GFX1132-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2
-; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1132-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc_lo
+; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
; GFX1132-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -12557,41 +14028,235 @@ define amdgpu_kernel void @uniform_fadd_v2bf16(ptr addrspace(1) %result, ptr add
; GFX1132-FAKE16-NEXT: buffer_store_b32 v2, off, s[8:11], 0
; GFX1132-FAKE16-NEXT: s_endpgm
;
-; GFX1264-LABEL: uniform_fadd_v2bf16:
-; GFX1264: ; %bb.0:
-; GFX1264-NEXT: s_clause 0x1
-; GFX1264-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX1264-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1264-NEXT: v_mov_b32_e32 v0, 0
-; GFX1264-NEXT: s_wait_kmcnt 0x0
-; GFX1264-NEXT: v_mov_b32_e32 v1, s6
-; GFX1264-NEXT: global_atomic_pk_add_bf16 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1264-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1264-NEXT: s_mov_b32 s2, -1
-; GFX1264-NEXT: s_wait_loadcnt 0x0
-; GFX1264-NEXT: buffer_store_b32 v0, off, s[0:3], null
-; GFX1264-NEXT: s_endpgm
+; GFX1264-TRUE16-LABEL: uniform_fadd_v2bf16:
+; GFX1264-TRUE16: ; %bb.0:
+; GFX1264-TRUE16-NEXT: s_clause 0x1
+; GFX1264-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1264-TRUE16-NEXT: s_load_b32 s4, s[4:5], 0x34
+; GFX1264-TRUE16-NEXT: s_mov_b64 s[8:9], 0
+; GFX1264-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1264-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1264-TRUE16-NEXT: s_load_b32 s5, s[2:3], 0x0
+; GFX1264-TRUE16-NEXT: s_and_b32 s10, s4, 0xffff0000
+; GFX1264-TRUE16-NEXT: s_lshl_b32 s11, s4, 16
+; GFX1264-TRUE16-NEXT: s_mov_b32 s4, s2
+; GFX1264-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1264-TRUE16-NEXT: s_mov_b32 s5, s3
+; GFX1264-TRUE16-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX1264-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1264-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX1264-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX1264-TRUE16-NEXT: v_add_f32_e32 v0, s11, v0
+; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1264-TRUE16-NEXT: v_add_f32_e32 v2, s10, v2
+; GFX1264-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1264-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX1264-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX1264-TRUE16-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX1264-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX1264-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1264-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX1264-TRUE16-NEXT: s_wait_alu 0xfffd
+; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1264-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
+; GFX1264-TRUE16-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX1264-TRUE16-NEXT: s_wait_alu 0xfffd
+; GFX1264-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc
+; GFX1264-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX1264-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1264-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX1264-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1264-TRUE16-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX1264-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-TRUE16-NEXT: s_and_not1_b64 exec, exec, s[8:9]
+; GFX1264-TRUE16-NEXT: s_cbranch_execnz .LBB21_1
+; GFX1264-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1264-TRUE16-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX1264-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1264-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX1264-TRUE16-NEXT: buffer_store_b32 v2, off, s[0:3], null
+; GFX1264-TRUE16-NEXT: s_endpgm
;
-; GFX1232-LABEL: uniform_fadd_v2bf16:
-; GFX1232: ; %bb.0:
-; GFX1232-NEXT: s_clause 0x1
-; GFX1232-NEXT: s_load_b32 s6, s[4:5], 0x34
-; GFX1232-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1232-NEXT: s_wait_kmcnt 0x0
-; GFX1232-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s6
-; GFX1232-NEXT: global_atomic_pk_add_bf16 v0, v0, v1, s[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1232-NEXT: s_mov_b32 s3, 0x31016000
-; GFX1232-NEXT: s_mov_b32 s2, -1
-; GFX1232-NEXT: s_wait_loadcnt 0x0
-; GFX1232-NEXT: buffer_store_b32 v0, off, s[0:3], null
-; GFX1232-NEXT: s_endpgm
- %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, <2 x bfloat> %val monotonic, align 4, !amdgpu.no.fine.grained.memory !0
+; GFX1264-FAKE16-LABEL: uniform_fadd_v2bf16:
+; GFX1264-FAKE16: ; %bb.0:
+; GFX1264-FAKE16-NEXT: s_clause 0x1
+; GFX1264-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
+; GFX1264-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x34
+; GFX1264-FAKE16-NEXT: s_mov_b64 s[2:3], 0
+; GFX1264-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1264-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1264-FAKE16-NEXT: s_load_b32 s1, s[10:11], 0x0
+; GFX1264-FAKE16-NEXT: s_lshl_b32 s12, s0, 16
+; GFX1264-FAKE16-NEXT: s_and_b32 s13, s0, 0xffff0000
+; GFX1264-FAKE16-NEXT: s_mov_b32 s4, s10
+; GFX1264-FAKE16-NEXT: s_mov_b32 s5, s11
+; GFX1264-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, s1
+; GFX1264-FAKE16-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX1264-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1264-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX1264-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX1264-FAKE16-NEXT: v_add_f32_e32 v0, s12, v0
+; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1264-FAKE16-NEXT: v_add_f32_e32 v2, s13, v2
+; GFX1264-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1264-FAKE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX1264-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX1264-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX1264-FAKE16-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX1264-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1264-FAKE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX1264-FAKE16-NEXT: v_cmp_u_f32_e64 s[0:1], v0, v0
+; GFX1264-FAKE16-NEXT: s_wait_alu 0xfffd
+; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1264-FAKE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc
+; GFX1264-FAKE16-NEXT: s_wait_alu 0xf1ff
+; GFX1264-FAKE16-NEXT: v_cndmask_b32_e64 v0, v3, v5, s[0:1]
+; GFX1264-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1264-FAKE16-NEXT: v_perm_b32 v0, v2, v0, 0x7060302
+; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX1264-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1264-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX1264-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; GFX1264-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1264-FAKE16-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX1264-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1264-FAKE16-NEXT: s_and_not1_b64 exec, exec, s[2:3]
+; GFX1264-FAKE16-NEXT: s_cbranch_execnz .LBB21_1
+; GFX1264-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1264-FAKE16-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX1264-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1264-FAKE16-NEXT: s_mov_b32 s10, -1
+; GFX1264-FAKE16-NEXT: buffer_store_b32 v2, off, s[8:11], null
+; GFX1264-FAKE16-NEXT: s_endpgm
+;
+; GFX1232-TRUE16-LABEL: uniform_fadd_v2bf16:
+; GFX1232-TRUE16: ; %bb.0:
+; GFX1232-TRUE16-NEXT: s_clause 0x1
+; GFX1232-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1232-TRUE16-NEXT: s_load_b32 s4, s[4:5], 0x34
+; GFX1232-TRUE16-NEXT: s_mov_b32 s8, 0
+; GFX1232-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1232-TRUE16-NEXT: s_load_b32 s5, s[2:3], 0x0
+; GFX1232-TRUE16-NEXT: s_and_b32 s9, s4, 0xffff0000
+; GFX1232-TRUE16-NEXT: s_lshl_b32 s10, s4, 16
+; GFX1232-TRUE16-NEXT: s_mov_b32 s4, s2
+; GFX1232-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, s5
+; GFX1232-TRUE16-NEXT: s_mov_b32 s5, s3
+; GFX1232-TRUE16-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX1232-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1232-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX1232-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX1232-TRUE16-NEXT: v_add_f32_e32 v0, s10, v0
+; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1232-TRUE16-NEXT: v_add_f32_e32 v2, s9, v2
+; GFX1232-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1232-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX1232-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX1232-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX1232-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX1232-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1232-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX1232-TRUE16-NEXT: s_wait_alu 0xfffd
+; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1232-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc_lo
+; GFX1232-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX1232-TRUE16-NEXT: s_wait_alu 0xfffd
+; GFX1232-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc_lo
+; GFX1232-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1232-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1232-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1232-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX1232-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1232-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1232-TRUE16-NEXT: s_or_b32 s8, vcc_lo, s8
+; GFX1232-TRUE16-NEXT: s_wait_alu 0xfffe
+; GFX1232-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s8
+; GFX1232-TRUE16-NEXT: s_cbranch_execnz .LBB21_1
+; GFX1232-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1232-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s8
+; GFX1232-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1232-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX1232-TRUE16-NEXT: buffer_store_b32 v2, off, s[0:3], null
+; GFX1232-TRUE16-NEXT: s_endpgm
+;
+; GFX1232-FAKE16-LABEL: uniform_fadd_v2bf16:
+; GFX1232-FAKE16: ; %bb.0:
+; GFX1232-FAKE16-NEXT: s_clause 0x1
+; GFX1232-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
+; GFX1232-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x34
+; GFX1232-FAKE16-NEXT: s_mov_b32 s1, 0
+; GFX1232-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
+; GFX1232-FAKE16-NEXT: s_mov_b32 s6, -1
+; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1232-FAKE16-NEXT: s_load_b32 s4, s[10:11], 0x0
+; GFX1232-FAKE16-NEXT: s_lshl_b32 s2, s0, 16
+; GFX1232-FAKE16-NEXT: s_and_b32 s3, s0, 0xffff0000
+; GFX1232-FAKE16-NEXT: s_mov_b32 s5, s11
+; GFX1232-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, s4
+; GFX1232-FAKE16-NEXT: s_mov_b32 s4, s10
+; GFX1232-FAKE16-NEXT: .LBB21_1: ; %atomicrmw.start
+; GFX1232-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1232-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX1232-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX1232-FAKE16-NEXT: v_add_f32_e32 v0, s2, v0
+; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1232-FAKE16-NEXT: v_add_f32_e32 v2, s3, v2
+; GFX1232-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1232-FAKE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX1232-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX1232-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX1232-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX1232-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1232-FAKE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX1232-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v0, v0
+; GFX1232-FAKE16-NEXT: s_wait_alu 0xfffd
+; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1232-FAKE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX1232-FAKE16-NEXT: s_wait_alu 0xf1ff
+; GFX1232-FAKE16-NEXT: v_cndmask_b32_e64 v0, v3, v5, s0
+; GFX1232-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-FAKE16-NEXT: v_perm_b32 v0, v2, v0, 0x7060302
+; GFX1232-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX1232-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1232-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX1232-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX1232-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX1232-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX1232-FAKE16-NEXT: s_wait_alu 0xfffe
+; GFX1232-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
+; GFX1232-FAKE16-NEXT: s_cbranch_execnz .LBB21_1
+; GFX1232-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1232-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
+; GFX1232-FAKE16-NEXT: s_mov_b32 s11, 0x31016000
+; GFX1232-FAKE16-NEXT: s_mov_b32 s10, -1
+; GFX1232-FAKE16-NEXT: buffer_store_b32 v2, off, s[8:11], null
+; GFX1232-FAKE16-NEXT: s_endpgm
+ %rmw = atomicrmw fadd ptr addrspace(1) %uniform.ptr, <2 x bfloat> %val monotonic, align 4
store <2 x bfloat> %rmw, ptr addrspace(1) %result
ret void
}
-
-!0 = !{}
-
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX1132_DPP-FAKE16: {{.*}}
; GFX1132_DPP-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index 10e523d1a0cf1..49fe1eed9c514 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -9826,13 +9826,12 @@ define <2 x bfloat> @v_fadd_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fadd_v2bf16:
@@ -10030,37 +10029,36 @@ define <3 x bfloat> @v_fadd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX11TRUE16-LABEL: v_fadd_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_add_f32 v1, v1, v3 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_dual_add_f32 v0, v0, v2 :: v_dual_lshlrev_b32 v3, 16, v3
; GFX11TRUE16-NEXT: v_add_f32_e32 v4, v5, v4
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v5, v6, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v4, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fadd_v3bf16:
@@ -10330,21 +10328,17 @@ define <4 x bfloat> @v_fadd_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v4
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v9, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v9, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v10, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v3, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fadd_v4bf16:
@@ -10772,81 +10766,77 @@ define <8 x bfloat> @v_fadd_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
+; GFX11TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
+; GFX11TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v3
-; GFX11TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_dual_add_f32 v8, v9, v8 :: v_dual_lshlrev_b32 v1, 16, v1
; GFX11TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_add_f32_e32 v7, v3, v7
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v5
; GFX11TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_add_f32 v2, v2, v6 :: v_dual_add_f32 v3, v3, v7
-; GFX11TRUE16-NEXT: v_add_f32_e32 v7, v10, v9
-; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v8
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add_f32_e32 v9, v10, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v8
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
; GFX11TRUE16-NEXT: v_add3_u32 v10, v11, v8, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v11, v3, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v12, v7, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v10, v9, vcc_lo
-; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v11, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v12, v7, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX11TRUE16-NEXT: v_bfe_u32 v12, v2, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_bfe_u32 v11, v7, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v12, v9, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v7
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc_lo
+; GFX11TRUE16-NEXT: v_add_f32_e32 v6, v2, v6
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v11, v7, 0x7fff
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v12, v9, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
+; GFX11TRUE16-NEXT: v_bfe_u32 v12, v6, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v2, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add3_u32 v7, v12, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v12, v6, 0x7fff
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v0
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v6, v10, v11 :: v_dual_lshlrev_b32 v11, 16, v4
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v10, v11, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v4
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v1, v1, v5
-; GFX11TRUE16-NEXT: v_dual_add_f32 v5, v12, v11 :: v_dual_cndmask_b32 v2, v7, v10
-; GFX11TRUE16-NEXT: v_add_f32_e32 v9, v14, v13
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v8
-; GFX11TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add_f32_e32 v5, v12, v11
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11TRUE16-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v9, v14, v13
; GFX11TRUE16-NEXT: v_bfe_u32 v4, v5, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11TRUE16-NEXT: v_bfe_u32 v13, v9, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v5, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v5
; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v13, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v12, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v13, v9, 0x7fff
; GFX11TRUE16-NEXT: v_bfe_u32 v13, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.h
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v14, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11TRUE16-NEXT: v_add3_u32 v7, v13, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v11, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v12, v13, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v10, v11, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v5
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v12, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v4, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v8.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v12, v13, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fadd_v8bf16:
@@ -11656,158 +11646,153 @@ define <16 x bfloat> @v_fadd_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX11TRUE16-LABEL: v_fadd_v16bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v14
; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v7
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_add_f32 v18, v19, v18 :: v_dual_and_b32 v23, 0xffff0000, v9
-; GFX11TRUE16-NEXT: v_dual_add_f32 v16, v17, v16 :: v_dual_lshlrev_b32 v15, 16, v15
-; GFX11TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v18
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_add_f32 v18, v19, v18 :: v_dual_and_b32 v21, 0xffff0000, v13
+; GFX11TRUE16-NEXT: v_add_f32_e32 v16, v17, v16
+; GFX11TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v5
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v16
+; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v18
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v16, 16, 1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11TRUE16-NEXT: v_dual_add_f32 v6, v6, v14 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_add_f32 v7, v7, v15 :: v_dual_and_b32 v14, 0xffff0000, v13
-; GFX11TRUE16-NEXT: v_bfe_u32 v15, v16, 16, 1
-; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v9
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v17, v7, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v7
-; GFX11TRUE16-NEXT: v_add3_u32 v15, v15, v16, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_dual_add_f32 v14, v6, v14 :: v_dual_add_f32 v15, v7, v15
+; GFX11TRUE16-NEXT: v_add3_u32 v7, v17, v16, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v16
; GFX11TRUE16-NEXT: v_bfe_u32 v16, v18, 16, 1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v7, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v15, v20, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v15, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v17, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v19, v15, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v15
; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v18, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v20, v6, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v17, v21, vcc_lo
-; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v5
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v17, v19, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v4
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_add_f32 v14, v17, v14 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v6
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v7.h
-; GFX11TRUE16-NEXT: v_add_f32_e32 v5, v5, v13
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v16, v19, vcc_lo
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v20, v6, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v18, v14, 16, 1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v12
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v5, 16, 1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v7, v15
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v16, v17, vcc_lo
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v18, v14, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v14
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v15.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v16, v20, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v16, v14, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v14
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11TRUE16-NEXT: v_add3_u32 v19, v19, v5, 0x7fff
-; GFX11TRUE16-NEXT: v_add_f32_e32 v4, v4, v12
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v14, v16, v17, vcc_lo
-; GFX11TRUE16-NEXT: v_add_f32_e32 v18, v21, v20
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v5
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11TRUE16-NEXT: v_bfe_u32 v17, v4, 16, 1
-; GFX11TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v6, v13
-; GFX11TRUE16-NEXT: v_bfe_u32 v21, v18, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v18
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v5, v19, v20 :: v_dual_and_b32 v20, 0xffff0000, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v14, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v14, v16, v20 :: v_dual_lshlrev_b32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v12
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v14.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_dual_add_f32 v12, v4, v12 :: v_dual_add_f32 v13, v5, v13
+; GFX11TRUE16-NEXT: v_dual_add_f32 v18, v19, v18 :: v_dual_add_f32 v17, v22, v21
+; GFX11TRUE16-NEXT: v_bfe_u32 v22, v12, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v13, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v20, v18, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v21, v17, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v17
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v20, v18, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_add3_u32 v5, v21, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v13
+; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v18
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v5, v5, v16 :: v_dual_and_b32 v20, 0xffff0000, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v16, v19, v13, 0x7fff
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v16, v17, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v12, v21, v18, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v22, v12, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v12
+; GFX11TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v21, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v11
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v12, v16, vcc_lo
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v17, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v4
-; GFX11TRUE16-NEXT: v_dual_add_f32 v18, v20, v19 :: v_dual_and_b32 v19, 0xffff0000, v10
-; GFX11TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v4, v16, v17 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v13.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v17, v18, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_add_f32_e32 v16, v20, v19
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v18
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_add_f32 v2, v2, v10 :: v_dual_add_f32 v3, v3, v11
-; GFX11TRUE16-NEXT: v_bfe_u32 v11, v18, 16, 1
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v12.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_bfe_u32 v19, v16, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v16
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v22, v2, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v17, v3, 16, 1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v19, v19, v16, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v11, v11, v18, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v17, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v3
-; GFX11TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v5, v14
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v12
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v10, v17, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v22, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v19, v16, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v10
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_dual_add_f32 v18, v21, v20 :: v_dual_lshlrev_b32 v11, 16, v11
+; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v9
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_bfe_u32 v20, v18, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v9
+; GFX11TRUE16-NEXT: v_dual_add_f32 v10, v2, v10 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_add_f32_e32 v11, v3, v11
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v16
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v11, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v11
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v20, v18, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v19, v11, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v18
+; GFX11TRUE16-NEXT: v_bfe_u32 v20, v10, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v2, v16, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11TRUE16-NEXT: v_add3_u32 v16, v20, v10, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v10
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v0
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v19, v21, vcc_lo
-; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v2
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v8
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v2, v17, v19 :: v_dual_lshlrev_b32 v19, 16, v8
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
; GFX11TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v16, v19, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v11.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add_f32_e32 v9, v20, v19
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v16, v18, vcc_lo
; GFX11TRUE16-NEXT: v_bfe_u32 v16, v1, 16, 1
-; GFX11TRUE16-NEXT: v_add_f32_e32 v9, v22, v21
-; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v8
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_dual_add_f32 v17, v22, v21 :: v_dual_add_f32 v0, v0, v8
; GFX11TRUE16-NEXT: v_bfe_u32 v8, v9, 16, 1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v16, v22, vcc_lo
-; GFX11TRUE16-NEXT: v_add_f32_e32 v17, v24, v23
+; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_bfe_u32 v21, v17, 16, 1
; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v17
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v20, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v23, v17, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v17
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v24, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v21, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v21, v0, 16, 1
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v22, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: v_add3_u32 v19, v23, v17, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v23, v0, 16, 1
-; GFX11TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v10
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v9, v19, v21, vcc_lo
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v23, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v20, v21, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v18, v19, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v9
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v16, v22, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v11, v20, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v8, v0
-; GFX11TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v11
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v16.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v20, v21, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fadd_v16bf16:
@@ -13510,303 +13495,274 @@ define <32 x bfloat> @v_fadd_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: scratch_load_b32 v32, off, s32
-; GFX11TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v21
-; GFX11TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v5
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v26
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; GFX11TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v19
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; GFX11TRUE16-NEXT: v_add_f32_e32 v5, v5, v21
-; GFX11TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v18
+; GFX11TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v20
+; GFX11TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v22
+; GFX11TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v6
+; GFX11TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v11
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_add_f32 v20, v4, v20 :: v_dual_lshlrev_b32 v11, 16, v11
+; GFX11TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v7
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX11TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v17
; GFX11TRUE16-NEXT: v_and_b32_e32 v84, 0xffff0000, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX11TRUE16-NEXT: v_bfe_u32 v103, v5, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v112, 0x400000, v5
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_and_b32_e32 v85, 0xffff0000, v16
-; GFX11TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v24
-; GFX11TRUE16-NEXT: v_add3_u32 v103, v103, v5, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v3
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v9
-; GFX11TRUE16-NEXT: v_dual_add_f32 v1, v1, v17 :: v_dual_lshlrev_b32 v24, 16, v24
-; GFX11TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v7
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add_f32_e32 v3, v3, v19
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v22
-; GFX11TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v11
-; GFX11TRUE16-NEXT: v_bfe_u32 v119, v3, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v128, 0x400000, v3
-; GFX11TRUE16-NEXT: v_bfe_u32 v135, v1, 16, 1
-; GFX11TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v25
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11TRUE16-NEXT: v_add3_u32 v119, v119, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v8
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11TRUE16-NEXT: v_or_b32_e32 v144, 0x400000, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v135, v135, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_dual_add_f32 v19, v82, v81 :: v_dual_lshlrev_b32 v18, 16, v18
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX11TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v129, v19, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v130, 0x400000, v19
-; GFX11TRUE16-NEXT: v_add_f32_e32 v2, v2, v18
-; GFX11TRUE16-NEXT: v_dual_add_f32 v18, v84, v83 :: v_dual_add_f32 v9, v9, v25
-; GFX11TRUE16-NEXT: v_add3_u32 v129, v129, v19, 0x7fff
; GFX11TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v131, v2, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v132, 0x400000, v2
-; GFX11TRUE16-NEXT: v_add_f32_e32 v17, v86, v85
-; GFX11TRUE16-NEXT: v_dual_add_f32 v8, v8, v24 :: v_dual_and_b32 v39, 0xffff0000, v27
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v131, v131, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v133, v18, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v145, v17, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v146, 0x400000, v17
-; GFX11TRUE16-NEXT: v_bfe_u32 v83, v8, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v84, 0x400000, v8
-; GFX11TRUE16-NEXT: v_or_b32_e32 v134, 0x400000, v18
-; GFX11TRUE16-NEXT: v_add3_u32 v145, v145, v17, 0x7fff
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v23
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX11TRUE16-NEXT: v_bfe_u32 v115, v20, 16, 1
; GFX11TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v10
-; GFX11TRUE16-NEXT: v_add3_u32 v83, v83, v8, 0x7fff
-; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v16
-; GFX11TRUE16-NEXT: v_dual_add_f32 v24, v64, v55 :: v_dual_and_b32 v37, 0xffff0000, v28
-; GFX11TRUE16-NEXT: v_add_f32_e32 v7, v7, v23
-; GFX11TRUE16-NEXT: v_dual_add_f32 v23, v66, v65 :: v_dual_lshlrev_b32 v28, 16, v28
-; GFX11TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v29
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v85, v24, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v86, 0x400000, v24
-; GFX11TRUE16-NEXT: v_bfe_u32 v97, v23, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v98, 0x400000, v23
-; GFX11TRUE16-NEXT: v_bfe_u32 v87, v7, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v85, v85, v24, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v20
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11TRUE16-NEXT: v_add3_u32 v97, v97, v23, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11TRUE16-NEXT: v_or_b32_e32 v96, 0x400000, v7
-; GFX11TRUE16-NEXT: v_add3_u32 v87, v87, v7, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11TRUE16-NEXT: v_add_f32_e32 v4, v4, v20
-; GFX11TRUE16-NEXT: v_add_f32_e32 v20, v80, v71
-; GFX11TRUE16-NEXT: v_bfe_u32 v71, v9, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v80, 0x400000, v9
-; GFX11TRUE16-NEXT: v_dual_add_f32 v21, v70, v69 :: v_dual_lshlrev_b32 v10, 16, v10
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v71, v71, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX11TRUE16-NEXT: v_dual_add_f32 v10, v10, v26 :: v_dual_lshlrev_b32 v27, 16, v27
-; GFX11TRUE16-NEXT: v_dual_add_f32 v26, v52, v51 :: v_dual_add_f32 v25, v54, v53
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add_f32_e32 v6, v6, v22
-; GFX11TRUE16-NEXT: v_dual_add_f32 v11, v11, v27 :: v_dual_and_b32 v36, 0xffff0000, v13
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v24
+; GFX11TRUE16-NEXT: v_dual_add_f32 v1, v1, v17 :: v_dual_lshlrev_b32 v24, 16, v24
+; GFX11TRUE16-NEXT: v_dual_add_f32 v23, v7, v23 :: v_dual_and_b32 v82, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_add_f32_e32 v22, v6, v22
+; GFX11TRUE16-NEXT: v_add_f32_e32 v17, v86, v85
+; GFX11TRUE16-NEXT: v_or_b32_e32 v116, 0x400000, v20
+; GFX11TRUE16-NEXT: v_add3_u32 v115, v115, v20, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v18
+; GFX11TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v8
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v27
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX11TRUE16-NEXT: v_bfe_u32 v87, v23, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v99, v22, 16, 1
+; GFX11TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v21
+; GFX11TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v5
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v19
+; GFX11TRUE16-NEXT: v_or_b32_e32 v96, 0x400000, v23
+; GFX11TRUE16-NEXT: v_or_b32_e32 v100, 0x400000, v22
+; GFX11TRUE16-NEXT: v_add3_u32 v87, v87, v23, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v3
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v99, v99, v22, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX11TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v26
+; GFX11TRUE16-NEXT: v_dual_add_f32 v21, v5, v21 :: v_dual_lshlrev_b32 v26, 16, v26
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_add_f32 v19, v3, v19 :: v_dual_and_b32 v52, 0xffff0000, v9
+; GFX11TRUE16-NEXT: v_dual_add_f32 v27, v11, v27 :: v_dual_and_b32 v36, 0xffff0000, v13
+; GFX11TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v25
+; GFX11TRUE16-NEXT: v_dual_add_f32 v26, v10, v26 :: v_dual_lshlrev_b32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_dual_add_f32 v24, v8, v24 :: v_dual_and_b32 v35, 0xffff0000, v29
; GFX11TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v30
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
; GFX11TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v14
-; GFX11TRUE16-NEXT: v_add_f32_e32 v22, v68, v67
-; GFX11TRUE16-NEXT: v_dual_add_f32 v27, v50, v49 :: v_dual_and_b32 v38, 0xffff0000, v12
+; GFX11TRUE16-NEXT: v_bfe_u32 v103, v21, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v119, v19, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_add_f32 v29, v13, v29 :: v_dual_and_b32 v38, 0xffff0000, v12
+; GFX11TRUE16-NEXT: v_dual_add_f32 v33, v34, v33 :: v_dual_lshlrev_b32 v30, 16, v30
+; GFX11TRUE16-NEXT: v_add_f32_e32 v34, v36, v35
+; GFX11TRUE16-NEXT: v_add_f32_e32 v36, v48, v39
+; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v16
+; GFX11TRUE16-NEXT: v_dual_add_f32 v48, v64, v55 :: v_dual_and_b32 v37, 0xffff0000, v28
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11TRUE16-NEXT: v_dual_add_f32 v13, v13, v29 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_add_f32 v29, v38, v37 :: v_dual_lshlrev_b32 v30, 16, v30
-; GFX11TRUE16-NEXT: v_and_b32_e32 v31, 0xffff0000, v15
-; GFX11TRUE16-NEXT: v_dual_add_f32 v12, v12, v28 :: v_dual_lshlrev_b32 v15, 16, v15
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add_f32_e32 v14, v14, v30
-; GFX11TRUE16-NEXT: v_add_f32_e32 v28, v48, v39
-; GFX11TRUE16-NEXT: v_dual_add_f32 v30, v36, v35 :: v_dual_add_f32 v33, v34, v33
-; GFX11TRUE16-NEXT: v_bfe_u32 v39, v13, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v35, v14, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v14
+; GFX11TRUE16-NEXT: v_or_b32_e32 v112, 0x400000, v21
+; GFX11TRUE16-NEXT: v_or_b32_e32 v128, 0x400000, v19
+; GFX11TRUE16-NEXT: v_add3_u32 v103, v103, v21, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX11TRUE16-NEXT: v_add3_u32 v119, v119, v19, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX11TRUE16-NEXT: v_add_f32_e32 v35, v38, v37
+; GFX11TRUE16-NEXT: v_dual_add_f32 v30, v14, v30 :: v_dual_and_b32 v31, 0xffff0000, v15
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v37, v30, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v16, v33, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v34, 0x400000, v33
+; GFX11TRUE16-NEXT: v_add_f32_e32 v18, v2, v18
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v33, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v33
+; GFX11TRUE16-NEXT: v_bfe_u32 v14, v27, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v30, 16, 1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX11TRUE16-NEXT: v_add3_u32 v35, v35, v14, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v30
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v33, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v37, v37, v30, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v48, 0x400000, v13
-; GFX11TRUE16-NEXT: v_bfe_u32 v49, v29, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v39, v39, v13, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v34, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11TRUE16-NEXT: v_or_b32_e32 v50, 0x400000, v29
-; GFX11TRUE16-NEXT: v_bfe_u32 v51, v12, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v49, v49, v29, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v52, 0x400000, v12
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v14, v35, v36, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v33, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v30
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v34, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v30, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v150, v14, v27, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v14, v2, v3, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11TRUE16-NEXT: v_bfe_u32 v53, v28, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v51, v51, v12, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v54, 0x400000, v28
-; GFX11TRUE16-NEXT: v_bfe_u32 v55, v11, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v30, v37, v38, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11TRUE16-NEXT: v_add3_u32 v53, v53, v28, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v64, 0x400000, v11
-; GFX11TRUE16-NEXT: v_bfe_u32 v65, v27, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v55, v55, v11, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v39, v48, vcc_lo
+; GFX11TRUE16-NEXT: v_dual_add_f32 v39, v54, v53 :: v_dual_lshlrev_b32 v12, 16, v12
+; GFX11TRUE16-NEXT: v_dual_add_f32 v54, v84, v83 :: v_dual_lshlrev_b32 v9, 16, v9
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v34
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v29, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v13, v36, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v34, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v30, v4, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
+; GFX11TRUE16-NEXT: v_add_f32_e32 v25, v9, v25
+; GFX11TRUE16-NEXT: v_dual_add_f32 v28, v12, v28 :: v_dual_lshlrev_b32 v15, 16, v15
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v29
+; GFX11TRUE16-NEXT: v_bfe_u32 v10, v35, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v29, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v149, v13, v36, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v6, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11TRUE16-NEXT: v_or_b32_e32 v66, 0x400000, v27
-; GFX11TRUE16-NEXT: v_bfe_u32 v67, v10, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v65, v65, v27, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v68, 0x400000, v10
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v29, v49, v50, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11TRUE16-NEXT: v_bfe_u32 v69, v26, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v67, v67, v10, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v70, 0x400000, v26
-; GFX11TRUE16-NEXT: v_bfe_u32 v81, v25, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v51, v52, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v35
+; GFX11TRUE16-NEXT: v_bfe_u32 v12, v28, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v10, v35, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v28
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v29, v8, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
+; GFX11TRUE16-NEXT: v_add3_u32 v148, v12, v28, 0x7fff
+; GFX11TRUE16-NEXT: v_add_f32_e32 v37, v50, v49
+; GFX11TRUE16-NEXT: v_or_b32_e32 v55, 0x400000, v36
+; GFX11TRUE16-NEXT: v_or_b32_e32 v64, 0x400000, v27
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v10, v11, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11TRUE16-NEXT: v_add3_u32 v69, v69, v26, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v82, 0x400000, v25
-; GFX11TRUE16-NEXT: v_add3_u32 v81, v81, v25, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v99, v6, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v28, v53, v54, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11TRUE16-NEXT: v_or_b32_e32 v100, 0x400000, v6
-; GFX11TRUE16-NEXT: v_bfe_u32 v101, v22, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v99, v99, v6, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v102, 0x400000, v22
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v55, v64, vcc_lo
+; GFX11TRUE16-NEXT: v_add_f32_e32 v49, v66, v65
+; GFX11TRUE16-NEXT: v_bfe_u32 v65, v37, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v66, 0x400000, v37
+; GFX11TRUE16-NEXT: v_add_f32_e32 v38, v52, v51
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v148, v16, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
+; GFX11TRUE16-NEXT: v_add3_u32 v65, v65, v37, 0x7fff
+; GFX11TRUE16-NEXT: v_add_f32_e32 v51, v70, v69
+; GFX11TRUE16-NEXT: v_bfe_u32 v69, v38, 16, 1
+; GFX11TRUE16-NEXT: v_add_f32_e32 v52, v80, v71
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v149, v55, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11TRUE16-NEXT: v_add3_u32 v101, v101, v22, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v113, v21, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v114, 0x400000, v21
-; GFX11TRUE16-NEXT: v_bfe_u32 v115, v4, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v27, v65, v66, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11TRUE16-NEXT: v_add3_u32 v113, v113, v21, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v116, 0x400000, v4
-; GFX11TRUE16-NEXT: v_bfe_u32 v117, v20, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v115, v115, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v67, v68, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v70, 0x400000, v38
+; GFX11TRUE16-NEXT: v_bfe_u32 v71, v25, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v69, v69, v38, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v80, 0x400000, v25
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v27, v150, v64, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
+; GFX11TRUE16-NEXT: v_add3_u32 v71, v71, v25, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v83, v24, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v84, 0x400000, v24
+; GFX11TRUE16-NEXT: v_bfe_u32 v85, v48, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v65, v66, vcc_lo
+; GFX11TRUE16-NEXT: v_add_f32_e32 v50, v68, v67
+; GFX11TRUE16-NEXT: v_bfe_u32 v67, v26, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v68, 0x400000, v26
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11TRUE16-NEXT: v_or_b32_e32 v118, 0x400000, v20
-; GFX11TRUE16-NEXT: v_add3_u32 v117, v117, v20, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v83, v83, v24, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v86, 0x400000, v48
+; GFX11TRUE16-NEXT: v_add3_u32 v67, v67, v26, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v85, v85, v48, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v97, v49, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v98, 0x400000, v49
+; GFX11TRUE16-NEXT: v_bfe_u32 v101, v50, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v26, v67, v68, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
+; GFX11TRUE16-NEXT: v_add3_u32 v97, v97, v49, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v102, 0x400000, v50
+; GFX11TRUE16-NEXT: v_add3_u32 v101, v101, v50, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v113, v51, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v9, v69, v70, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11TRUE16-NEXT: v_or_b32_e32 v114, 0x400000, v51
+; GFX11TRUE16-NEXT: v_bfe_u32 v117, v52, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v113, v113, v51, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v118, 0x400000, v52
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v25, v71, v80, vcc_lo
+; GFX11TRUE16-NEXT: v_add_f32_e32 v53, v82, v81
+; GFX11TRUE16-NEXT: v_bfe_u32 v81, v39, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v82, 0x400000, v39
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v39, v39
+; GFX11TRUE16-NEXT: v_add3_u32 v117, v117, v52, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v129, v53, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v81, v81, v39, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v130, 0x400000, v53
+; GFX11TRUE16-NEXT: v_bfe_u32 v131, v18, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v132, 0x400000, v18
+; GFX11TRUE16-NEXT: v_add3_u32 v129, v129, v53, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v81, v82, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11TRUE16-NEXT: v_bfe_u32 v135, v1, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v131, v131, v18, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v144, 0x400000, v1
; GFX11TRUE16-NEXT: v_bfe_u32 v147, v0, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v24, v83, v84, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
+; GFX11TRUE16-NEXT: v_add3_u32 v135, v135, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v133, v54, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v26, v69, v70, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11TRUE16-NEXT: v_add3_u32 v133, v133, v18, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v147, v147, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v10.l, v10.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v11.l, v11.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v9, v71, v80, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v12.l, v12.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v13.l, v13.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v14.l, v14.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v9.l, v9.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v25, v81, v82, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v10, v27
-; GFX11TRUE16-NEXT: v_bfi_b32 v11, 0xffff, v11, v28
-; GFX11TRUE16-NEXT: v_bfi_b32 v9, 0xffff, v9, v26
-; GFX11TRUE16-NEXT: v_bfi_b32 v12, 0xffff, v12, v29
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v83, v84, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11TRUE16-NEXT: v_bfi_b32 v13, 0xffff, v13, v30
-; GFX11TRUE16-NEXT: v_bfi_b32 v14, 0xffff, v14, v16
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v24, v85, v86, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11TRUE16-NEXT: v_bfi_b32 v8, 0xffff, v8, v25
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v87, v96, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v85, v86, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v7.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v23, v97, v98, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v7, v24
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v99, v100, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v134, 0x400000, v54
+; GFX11TRUE16-NEXT: v_bfe_u32 v145, v17, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v133, v133, v54, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v146, 0x400000, v17
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v23, v87, v96, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v49, v49
+; GFX11TRUE16-NEXT: v_add3_u32 v145, v145, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v12.l, v16.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v8.l, v24.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v23.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v97, v98, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v22, v101, v102, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v6, v23
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v103, v112, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v9.l, v25.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v10.l, v26.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v11.l, v27.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v13.l, v29.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v22, v99, v100, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v50, v50
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v14.l, v30.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v22.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v101, v102, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v21, v113, v114, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v5, v22
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v115, v116, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v21, v103, v112, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v51, v51
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v21.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v113, v114, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v20, v117, v118, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v21
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v119, v128, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v20, v115, v116, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v52, v52
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v20.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v117, v118, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v19, v129, v130, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v20
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v131, v132, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v19, v119, v128, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v53, v53
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v19.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v129, v130, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v131, v132, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v135, v144, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v18.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v28, v135, v144, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v19
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v147, v33, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v133, v134, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v33, v147, v33, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v54, v54
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v133, v134, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v18
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v17, v145, v146, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v28.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v145, v146, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v33.h
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v32
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v17
-; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v32
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_add_f32_e32 v15, v15, v33
-; GFX11TRUE16-NEXT: v_add_f32_e32 v17, v31, v17
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v32
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_dual_add_f32 v15, v15, v17 :: v_dual_and_b32 v18, 0xffff0000, v32
+; GFX11TRUE16-NEXT: v_add_f32_e32 v17, v31, v18
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_bfe_u32 v18, v15, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v17, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v15
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v17, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v17
+; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v17
; GFX11TRUE16-NEXT: v_add3_u32 v18, v18, v15, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_add3_u32 v19, v19, v17, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v18, v20, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v18, v20, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v15.l, v15.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v17, v19, v21, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v15, 0xffff, v15, v17
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v19, v16, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v15.l, v18.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fadd_v32bf16:
@@ -14550,13 +14506,12 @@ define <2 x bfloat> @v_fsub_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fsub_v2bf16:
@@ -14754,37 +14709,36 @@ define <3 x bfloat> @v_fsub_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX11TRUE16-LABEL: v_fsub_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_sub_f32 v1, v1, v3 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_sub_f32_e32 v0, v0, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_dual_sub_f32 v0, v0, v2 :: v_dual_lshlrev_b32 v3, 16, v3
; GFX11TRUE16-NEXT: v_sub_f32_e32 v4, v5, v4
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v5, v6, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_sub_f32_e32 v1, v1, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v4, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fsub_v3bf16:
@@ -15054,21 +15008,17 @@ define <4 x bfloat> @v_fsub_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v4
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v9, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v9, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v10, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v3, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fsub_v4bf16:
@@ -15368,13 +15318,12 @@ define <2 x bfloat> @v_fmul_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fmul_v2bf16:
@@ -15572,37 +15521,36 @@ define <3 x bfloat> @v_fmul_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX11TRUE16-LABEL: v_fmul_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v1, v1, v3 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, v0, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v0, v0, v2 :: v_dual_lshlrev_b32 v3, 16, v3
; GFX11TRUE16-NEXT: v_mul_f32_e32 v4, v5, v4
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v5, v6, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v4, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fmul_v3bf16:
@@ -15872,21 +15820,17 @@ define <4 x bfloat> @v_fmul_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v4
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v9, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v9, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v10, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v3, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fmul_v4bf16:
@@ -16314,81 +16258,77 @@ define <8 x bfloat> @v_fmul_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
+; GFX11TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
+; GFX11TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v3
-; GFX11TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_dual_mul_f32 v8, v9, v8 :: v_dual_lshlrev_b32 v1, 16, v1
; GFX11TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v7, v3, v7
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, v1, v5
; GFX11TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v2, v2, v6 :: v_dual_mul_f32 v3, v3, v7
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v7, v10, v9
-; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v8
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v9, v10, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v8
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
; GFX11TRUE16-NEXT: v_add3_u32 v10, v11, v8, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v11, v3, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v12, v7, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v10, v9, vcc_lo
-; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v11, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v12, v7, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX11TRUE16-NEXT: v_bfe_u32 v12, v2, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_bfe_u32 v11, v7, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v12, v9, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v7
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc_lo
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v6, v2, v6
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v11, v7, 0x7fff
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v12, v9, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
+; GFX11TRUE16-NEXT: v_bfe_u32 v12, v6, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v2, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add3_u32 v7, v12, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v12, v6, 0x7fff
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v0
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v6, v10, v11 :: v_dual_lshlrev_b32 v11, 16, v4
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v10, v11, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v4
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v0, v0, v4 :: v_dual_mul_f32 v1, v1, v5
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v5, v12, v11 :: v_dual_cndmask_b32 v2, v7, v10
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v9, v14, v13
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v8
-; GFX11TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v5, v12, v11
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v0, v0, v4 :: v_dual_mul_f32 v9, v14, v13
; GFX11TRUE16-NEXT: v_bfe_u32 v4, v5, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11TRUE16-NEXT: v_bfe_u32 v13, v9, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v5, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v5
; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v13, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v12, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v13, v9, 0x7fff
; GFX11TRUE16-NEXT: v_bfe_u32 v13, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.h
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v14, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11TRUE16-NEXT: v_add3_u32 v7, v13, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v11, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v12, v13, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v10, v11, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v5
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v12, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v4, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v8.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v12, v13, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fmul_v8bf16:
@@ -17198,158 +17138,153 @@ define <16 x bfloat> @v_fmul_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX11TRUE16-LABEL: v_fmul_v16bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v14
; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v7
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v18, v19, v18 :: v_dual_and_b32 v23, 0xffff0000, v9
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v16, v17, v16 :: v_dual_lshlrev_b32 v15, 16, v15
-; GFX11TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v18
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v18, v19, v18 :: v_dual_and_b32 v21, 0xffff0000, v13
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v16, v17, v16
+; GFX11TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v5
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v16
+; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v18
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v16, 16, 1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v6, v6, v14 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v7, v7, v15 :: v_dual_and_b32 v14, 0xffff0000, v13
-; GFX11TRUE16-NEXT: v_bfe_u32 v15, v16, 16, 1
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, v1, v9
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v17, v7, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v7
-; GFX11TRUE16-NEXT: v_add3_u32 v15, v15, v16, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v14, v6, v14 :: v_dual_mul_f32 v15, v7, v15
+; GFX11TRUE16-NEXT: v_add3_u32 v7, v17, v16, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v16
; GFX11TRUE16-NEXT: v_bfe_u32 v16, v18, 16, 1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v7, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v15, v20, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v15, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v17, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v19, v15, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v15
; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v18, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v20, v6, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v17, v21, vcc_lo
-; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v5
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v17, v19, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v4
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v14, v17, v14 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v6
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v7.h
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v5, v5, v13
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v16, v19, vcc_lo
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v20, v6, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v18, v14, 16, 1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v12
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v5, 16, 1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v7, v15
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v16, v17, vcc_lo
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v18, v14, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v14
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v15.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v16, v20, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v16, v14, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v14
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11TRUE16-NEXT: v_add3_u32 v19, v19, v5, 0x7fff
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v4, v4, v12
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v14, v16, v17, vcc_lo
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v18, v21, v20
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v5
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11TRUE16-NEXT: v_bfe_u32 v17, v4, 16, 1
-; GFX11TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v6, v13
-; GFX11TRUE16-NEXT: v_bfe_u32 v21, v18, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v18
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v5, v19, v20 :: v_dual_and_b32 v20, 0xffff0000, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v14, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v14, v16, v20 :: v_dual_lshlrev_b32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v12
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v14.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v12, v4, v12 :: v_dual_mul_f32 v13, v5, v13
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v18, v19, v18 :: v_dual_mul_f32 v17, v22, v21
+; GFX11TRUE16-NEXT: v_bfe_u32 v22, v12, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v13, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v20, v18, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v21, v17, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v17
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v20, v18, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_add3_u32 v5, v21, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v13
+; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v18
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v5, v5, v16 :: v_dual_and_b32 v20, 0xffff0000, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v16, v19, v13, 0x7fff
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v16, v17, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v12, v21, v18, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v22, v12, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v12
+; GFX11TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v21, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v11
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v12, v16, vcc_lo
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v17, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v4
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v18, v20, v19 :: v_dual_and_b32 v19, 0xffff0000, v10
-; GFX11TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v4, v16, v17 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v13.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v17, v18, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_mul_f32_e32 v16, v20, v19
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v18
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v2, v2, v10 :: v_dual_mul_f32 v3, v3, v11
-; GFX11TRUE16-NEXT: v_bfe_u32 v11, v18, 16, 1
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v12.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_bfe_u32 v19, v16, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v16
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v22, v2, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v17, v3, 16, 1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v19, v19, v16, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v11, v11, v18, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v17, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v3
-; GFX11TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v5, v14
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v12
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v10, v17, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v22, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v19, v16, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v10
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v18, v21, v20 :: v_dual_lshlrev_b32 v11, 16, v11
+; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v9
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_bfe_u32 v20, v18, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, v1, v9
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v10, v2, v10 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v11, v3, v11
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v16
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v11, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v11
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v20, v18, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v19, v11, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v18
+; GFX11TRUE16-NEXT: v_bfe_u32 v20, v10, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v2, v16, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11TRUE16-NEXT: v_add3_u32 v16, v20, v10, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v10
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v0
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v19, v21, vcc_lo
-; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v2
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v8
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v2, v17, v19 :: v_dual_lshlrev_b32 v19, 16, v8
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
; GFX11TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v16, v19, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v11.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v9, v20, v19
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v16, v18, vcc_lo
; GFX11TRUE16-NEXT: v_bfe_u32 v16, v1, 16, 1
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v9, v22, v21
-; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, v0, v8
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v17, v22, v21 :: v_dual_mul_f32 v0, v0, v8
; GFX11TRUE16-NEXT: v_bfe_u32 v8, v9, 16, 1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v16, v22, vcc_lo
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v17, v24, v23
+; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_bfe_u32 v21, v17, 16, 1
; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v17
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v20, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v23, v17, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v17
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v24, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v21, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v21, v0, 16, 1
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v22, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: v_add3_u32 v19, v23, v17, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v23, v0, 16, 1
-; GFX11TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v10
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v9, v19, v21, vcc_lo
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v23, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v20, v21, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v18, v19, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v9
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v16, v22, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v11, v20, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v8, v0
-; GFX11TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v11
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v16.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v20, v21, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fmul_v16bf16:
@@ -19052,303 +18987,274 @@ define <32 x bfloat> @v_fmul_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: scratch_load_b32 v32, off, s32
-; GFX11TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v21
-; GFX11TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v5
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v26
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; GFX11TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v19
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v5, v5, v21
-; GFX11TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v18
+; GFX11TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v20
+; GFX11TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v22
+; GFX11TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v6
+; GFX11TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v11
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v20, v4, v20 :: v_dual_lshlrev_b32 v11, 16, v11
+; GFX11TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v7
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX11TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v17
; GFX11TRUE16-NEXT: v_and_b32_e32 v84, 0xffff0000, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX11TRUE16-NEXT: v_bfe_u32 v103, v5, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v112, 0x400000, v5
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_and_b32_e32 v85, 0xffff0000, v16
-; GFX11TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v24
-; GFX11TRUE16-NEXT: v_add3_u32 v103, v103, v5, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v3
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v9
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v1, v1, v17 :: v_dual_lshlrev_b32 v24, 16, v24
-; GFX11TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v7
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v3, v3, v19
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v22
-; GFX11TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v11
-; GFX11TRUE16-NEXT: v_bfe_u32 v119, v3, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v128, 0x400000, v3
-; GFX11TRUE16-NEXT: v_bfe_u32 v135, v1, 16, 1
-; GFX11TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v25
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11TRUE16-NEXT: v_add3_u32 v119, v119, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v8
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11TRUE16-NEXT: v_or_b32_e32 v144, 0x400000, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v135, v135, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v19, v82, v81 :: v_dual_lshlrev_b32 v18, 16, v18
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX11TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v129, v19, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v130, 0x400000, v19
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v2, v2, v18
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v18, v84, v83 :: v_dual_mul_f32 v9, v9, v25
-; GFX11TRUE16-NEXT: v_add3_u32 v129, v129, v19, 0x7fff
; GFX11TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v131, v2, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v132, 0x400000, v2
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v17, v86, v85
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v8, v8, v24 :: v_dual_and_b32 v39, 0xffff0000, v27
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v131, v131, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v133, v18, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v145, v17, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v146, 0x400000, v17
-; GFX11TRUE16-NEXT: v_bfe_u32 v83, v8, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v84, 0x400000, v8
-; GFX11TRUE16-NEXT: v_or_b32_e32 v134, 0x400000, v18
-; GFX11TRUE16-NEXT: v_add3_u32 v145, v145, v17, 0x7fff
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v23
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX11TRUE16-NEXT: v_bfe_u32 v115, v20, 16, 1
; GFX11TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v10
-; GFX11TRUE16-NEXT: v_add3_u32 v83, v83, v8, 0x7fff
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, v0, v16
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v24, v64, v55 :: v_dual_and_b32 v37, 0xffff0000, v28
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v7, v7, v23
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v23, v66, v65 :: v_dual_lshlrev_b32 v28, 16, v28
-; GFX11TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v29
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v85, v24, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v86, 0x400000, v24
-; GFX11TRUE16-NEXT: v_bfe_u32 v97, v23, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v98, 0x400000, v23
-; GFX11TRUE16-NEXT: v_bfe_u32 v87, v7, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v85, v85, v24, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v20
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11TRUE16-NEXT: v_add3_u32 v97, v97, v23, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11TRUE16-NEXT: v_or_b32_e32 v96, 0x400000, v7
-; GFX11TRUE16-NEXT: v_add3_u32 v87, v87, v7, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v4, v4, v20
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v20, v80, v71
-; GFX11TRUE16-NEXT: v_bfe_u32 v71, v9, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v80, 0x400000, v9
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v21, v70, v69 :: v_dual_lshlrev_b32 v10, 16, v10
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v71, v71, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v10, v10, v26 :: v_dual_lshlrev_b32 v27, 16, v27
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v26, v52, v51 :: v_dual_mul_f32 v25, v54, v53
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v6, v6, v22
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v11, v11, v27 :: v_dual_and_b32 v36, 0xffff0000, v13
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v24
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v1, v1, v17 :: v_dual_lshlrev_b32 v24, 16, v24
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v23, v7, v23 :: v_dual_and_b32 v82, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v22, v6, v22
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v17, v86, v85
+; GFX11TRUE16-NEXT: v_or_b32_e32 v116, 0x400000, v20
+; GFX11TRUE16-NEXT: v_add3_u32 v115, v115, v20, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v18
+; GFX11TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v8
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v27
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX11TRUE16-NEXT: v_bfe_u32 v87, v23, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v99, v22, 16, 1
+; GFX11TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v21
+; GFX11TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v5
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v19
+; GFX11TRUE16-NEXT: v_or_b32_e32 v96, 0x400000, v23
+; GFX11TRUE16-NEXT: v_or_b32_e32 v100, 0x400000, v22
+; GFX11TRUE16-NEXT: v_add3_u32 v87, v87, v23, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v3
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v99, v99, v22, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX11TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v26
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v21, v5, v21 :: v_dual_lshlrev_b32 v26, 16, v26
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v19, v3, v19 :: v_dual_and_b32 v52, 0xffff0000, v9
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v27, v11, v27 :: v_dual_and_b32 v36, 0xffff0000, v13
+; GFX11TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v25
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v26, v10, v26 :: v_dual_lshlrev_b32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v24, v8, v24 :: v_dual_and_b32 v35, 0xffff0000, v29
; GFX11TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v30
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
; GFX11TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v14
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v22, v68, v67
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v27, v50, v49 :: v_dual_and_b32 v38, 0xffff0000, v12
+; GFX11TRUE16-NEXT: v_bfe_u32 v103, v21, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v119, v19, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v29, v13, v29 :: v_dual_and_b32 v38, 0xffff0000, v12
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v33, v34, v33 :: v_dual_lshlrev_b32 v30, 16, v30
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v34, v36, v35
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v36, v48, v39
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, v0, v16
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v48, v64, v55 :: v_dual_and_b32 v37, 0xffff0000, v28
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v13, v13, v29 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v29, v38, v37 :: v_dual_lshlrev_b32 v30, 16, v30
-; GFX11TRUE16-NEXT: v_and_b32_e32 v31, 0xffff0000, v15
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v12, v12, v28 :: v_dual_lshlrev_b32 v15, 16, v15
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v14, v14, v30
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v28, v48, v39
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v30, v36, v35 :: v_dual_mul_f32 v33, v34, v33
-; GFX11TRUE16-NEXT: v_bfe_u32 v39, v13, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v35, v14, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v14
+; GFX11TRUE16-NEXT: v_or_b32_e32 v112, 0x400000, v21
+; GFX11TRUE16-NEXT: v_or_b32_e32 v128, 0x400000, v19
+; GFX11TRUE16-NEXT: v_add3_u32 v103, v103, v21, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX11TRUE16-NEXT: v_add3_u32 v119, v119, v19, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v35, v38, v37
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v30, v14, v30 :: v_dual_and_b32 v31, 0xffff0000, v15
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v37, v30, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v16, v33, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v34, 0x400000, v33
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v18, v2, v18
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v33, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v33
+; GFX11TRUE16-NEXT: v_bfe_u32 v14, v27, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v30, 16, 1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX11TRUE16-NEXT: v_add3_u32 v35, v35, v14, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v30
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v33, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v37, v37, v30, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v48, 0x400000, v13
-; GFX11TRUE16-NEXT: v_bfe_u32 v49, v29, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v39, v39, v13, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v34, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11TRUE16-NEXT: v_or_b32_e32 v50, 0x400000, v29
-; GFX11TRUE16-NEXT: v_bfe_u32 v51, v12, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v49, v49, v29, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v52, 0x400000, v12
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v14, v35, v36, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v33, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v30
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v34, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v30, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v150, v14, v27, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v14, v2, v3, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11TRUE16-NEXT: v_bfe_u32 v53, v28, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v51, v51, v12, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v54, 0x400000, v28
-; GFX11TRUE16-NEXT: v_bfe_u32 v55, v11, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v30, v37, v38, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11TRUE16-NEXT: v_add3_u32 v53, v53, v28, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v64, 0x400000, v11
-; GFX11TRUE16-NEXT: v_bfe_u32 v65, v27, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v55, v55, v11, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v39, v48, vcc_lo
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v39, v54, v53 :: v_dual_lshlrev_b32 v12, 16, v12
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v54, v84, v83 :: v_dual_lshlrev_b32 v9, 16, v9
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v34
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v29, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v13, v36, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v34, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v30, v4, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v25, v9, v25
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v28, v12, v28 :: v_dual_lshlrev_b32 v15, 16, v15
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v29
+; GFX11TRUE16-NEXT: v_bfe_u32 v10, v35, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v29, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v149, v13, v36, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v6, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11TRUE16-NEXT: v_or_b32_e32 v66, 0x400000, v27
-; GFX11TRUE16-NEXT: v_bfe_u32 v67, v10, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v65, v65, v27, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v68, 0x400000, v10
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v29, v49, v50, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11TRUE16-NEXT: v_bfe_u32 v69, v26, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v67, v67, v10, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v70, 0x400000, v26
-; GFX11TRUE16-NEXT: v_bfe_u32 v81, v25, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v51, v52, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v35
+; GFX11TRUE16-NEXT: v_bfe_u32 v12, v28, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v10, v35, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v28
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v29, v8, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
+; GFX11TRUE16-NEXT: v_add3_u32 v148, v12, v28, 0x7fff
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v37, v50, v49
+; GFX11TRUE16-NEXT: v_or_b32_e32 v55, 0x400000, v36
+; GFX11TRUE16-NEXT: v_or_b32_e32 v64, 0x400000, v27
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v10, v11, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11TRUE16-NEXT: v_add3_u32 v69, v69, v26, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v82, 0x400000, v25
-; GFX11TRUE16-NEXT: v_add3_u32 v81, v81, v25, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v99, v6, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v28, v53, v54, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11TRUE16-NEXT: v_or_b32_e32 v100, 0x400000, v6
-; GFX11TRUE16-NEXT: v_bfe_u32 v101, v22, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v99, v99, v6, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v102, 0x400000, v22
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v55, v64, vcc_lo
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v49, v66, v65
+; GFX11TRUE16-NEXT: v_bfe_u32 v65, v37, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v66, 0x400000, v37
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v38, v52, v51
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v148, v16, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
+; GFX11TRUE16-NEXT: v_add3_u32 v65, v65, v37, 0x7fff
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v51, v70, v69
+; GFX11TRUE16-NEXT: v_bfe_u32 v69, v38, 16, 1
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v52, v80, v71
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v149, v55, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11TRUE16-NEXT: v_add3_u32 v101, v101, v22, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v113, v21, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v114, 0x400000, v21
-; GFX11TRUE16-NEXT: v_bfe_u32 v115, v4, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v27, v65, v66, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11TRUE16-NEXT: v_add3_u32 v113, v113, v21, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v116, 0x400000, v4
-; GFX11TRUE16-NEXT: v_bfe_u32 v117, v20, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v115, v115, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v67, v68, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v70, 0x400000, v38
+; GFX11TRUE16-NEXT: v_bfe_u32 v71, v25, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v69, v69, v38, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v80, 0x400000, v25
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v27, v150, v64, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
+; GFX11TRUE16-NEXT: v_add3_u32 v71, v71, v25, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v83, v24, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v84, 0x400000, v24
+; GFX11TRUE16-NEXT: v_bfe_u32 v85, v48, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v65, v66, vcc_lo
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v50, v68, v67
+; GFX11TRUE16-NEXT: v_bfe_u32 v67, v26, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v68, 0x400000, v26
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11TRUE16-NEXT: v_or_b32_e32 v118, 0x400000, v20
-; GFX11TRUE16-NEXT: v_add3_u32 v117, v117, v20, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v83, v83, v24, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v86, 0x400000, v48
+; GFX11TRUE16-NEXT: v_add3_u32 v67, v67, v26, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v85, v85, v48, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v97, v49, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v98, 0x400000, v49
+; GFX11TRUE16-NEXT: v_bfe_u32 v101, v50, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v26, v67, v68, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
+; GFX11TRUE16-NEXT: v_add3_u32 v97, v97, v49, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v102, 0x400000, v50
+; GFX11TRUE16-NEXT: v_add3_u32 v101, v101, v50, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v113, v51, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v9, v69, v70, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11TRUE16-NEXT: v_or_b32_e32 v114, 0x400000, v51
+; GFX11TRUE16-NEXT: v_bfe_u32 v117, v52, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v113, v113, v51, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v118, 0x400000, v52
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v25, v71, v80, vcc_lo
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v53, v82, v81
+; GFX11TRUE16-NEXT: v_bfe_u32 v81, v39, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v82, 0x400000, v39
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v39, v39
+; GFX11TRUE16-NEXT: v_add3_u32 v117, v117, v52, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v129, v53, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v81, v81, v39, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v130, 0x400000, v53
+; GFX11TRUE16-NEXT: v_bfe_u32 v131, v18, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v132, 0x400000, v18
+; GFX11TRUE16-NEXT: v_add3_u32 v129, v129, v53, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v81, v82, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11TRUE16-NEXT: v_bfe_u32 v135, v1, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v131, v131, v18, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v144, 0x400000, v1
; GFX11TRUE16-NEXT: v_bfe_u32 v147, v0, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v24, v83, v84, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
+; GFX11TRUE16-NEXT: v_add3_u32 v135, v135, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v133, v54, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v26, v69, v70, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11TRUE16-NEXT: v_add3_u32 v133, v133, v18, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v147, v147, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v10.l, v10.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v11.l, v11.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v9, v71, v80, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v12.l, v12.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v13.l, v13.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v14.l, v14.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v9.l, v9.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v25, v81, v82, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v10, v27
-; GFX11TRUE16-NEXT: v_bfi_b32 v11, 0xffff, v11, v28
-; GFX11TRUE16-NEXT: v_bfi_b32 v9, 0xffff, v9, v26
-; GFX11TRUE16-NEXT: v_bfi_b32 v12, 0xffff, v12, v29
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v83, v84, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11TRUE16-NEXT: v_bfi_b32 v13, 0xffff, v13, v30
-; GFX11TRUE16-NEXT: v_bfi_b32 v14, 0xffff, v14, v16
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v24, v85, v86, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11TRUE16-NEXT: v_bfi_b32 v8, 0xffff, v8, v25
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v87, v96, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v85, v86, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v7.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v23, v97, v98, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v7, v24
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v99, v100, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v134, 0x400000, v54
+; GFX11TRUE16-NEXT: v_bfe_u32 v145, v17, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v133, v133, v54, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v146, 0x400000, v17
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v23, v87, v96, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v49, v49
+; GFX11TRUE16-NEXT: v_add3_u32 v145, v145, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v12.l, v16.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v8.l, v24.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v23.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v97, v98, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v22, v101, v102, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v6, v23
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v103, v112, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v9.l, v25.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v10.l, v26.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v11.l, v27.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v13.l, v29.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v22, v99, v100, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v50, v50
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v14.l, v30.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v22.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v101, v102, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v21, v113, v114, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v5, v22
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v115, v116, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v21, v103, v112, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v51, v51
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v21.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v113, v114, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v20, v117, v118, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v21
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v119, v128, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v20, v115, v116, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v52, v52
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v20.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v117, v118, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v19, v129, v130, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v20
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v131, v132, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v19, v119, v128, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v53, v53
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v19.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v129, v130, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v131, v132, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v135, v144, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v18.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v28, v135, v144, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v19
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v147, v33, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v133, v134, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v33, v147, v33, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v54, v54
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v133, v134, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v18
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v17, v145, v146, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v28.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v145, v146, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v33.h
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v32
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v17
-; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v32
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v15, v15, v33
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v17, v31, v17
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v32
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v15, v15, v17 :: v_dual_and_b32 v18, 0xffff0000, v32
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v17, v31, v18
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_bfe_u32 v18, v15, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v17, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v15
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v17, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v17
+; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v17
; GFX11TRUE16-NEXT: v_add3_u32 v18, v18, v15, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_add3_u32 v19, v19, v17, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v18, v20, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v18, v20, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v15.l, v15.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v17, v19, v21, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v15, 0xffff, v15, v17
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v19, v16, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v15.l, v18.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fmul_v32bf16:
@@ -20406,13 +20312,12 @@ define <2 x bfloat> @v_minnum_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_minnum_v2bf16:
@@ -20610,37 +20515,36 @@ define <3 x bfloat> @v_minnum_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX11TRUE16-LABEL: v_minnum_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_min_f32 v1, v1, v3 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_min_f32_e32 v0, v0, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_dual_min_f32 v0, v0, v2 :: v_dual_lshlrev_b32 v3, 16, v3
; GFX11TRUE16-NEXT: v_min_f32_e32 v4, v5, v4
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v5, v6, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_min_f32_e32 v1, v1, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v4, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_minnum_v3bf16:
@@ -20910,21 +20814,17 @@ define <4 x bfloat> @v_minnum_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v4
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v9, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v9, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v10, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v3, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_minnum_v4bf16:
@@ -21352,81 +21252,77 @@ define <8 x bfloat> @v_minnum_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
+; GFX11TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
+; GFX11TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v3
-; GFX11TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_dual_min_f32 v8, v9, v8 :: v_dual_lshlrev_b32 v1, 16, v1
; GFX11TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_min_f32_e32 v7, v3, v7
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_min_f32_e32 v1, v1, v5
; GFX11TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_min_f32 v2, v2, v6 :: v_dual_min_f32 v3, v3, v7
-; GFX11TRUE16-NEXT: v_min_f32_e32 v7, v10, v9
-; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v8
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_min_f32_e32 v9, v10, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v8
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
; GFX11TRUE16-NEXT: v_add3_u32 v10, v11, v8, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v11, v3, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v12, v7, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v10, v9, vcc_lo
-; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v11, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v12, v7, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX11TRUE16-NEXT: v_bfe_u32 v12, v2, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_bfe_u32 v11, v7, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v12, v9, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v7
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc_lo
+; GFX11TRUE16-NEXT: v_min_f32_e32 v6, v2, v6
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v11, v7, 0x7fff
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v12, v9, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
+; GFX11TRUE16-NEXT: v_bfe_u32 v12, v6, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v2, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add3_u32 v7, v12, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v12, v6, 0x7fff
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v0
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v6, v10, v11 :: v_dual_lshlrev_b32 v11, 16, v4
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v10, v11, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v4
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_min_f32 v0, v0, v4 :: v_dual_min_f32 v1, v1, v5
-; GFX11TRUE16-NEXT: v_dual_min_f32 v5, v12, v11 :: v_dual_cndmask_b32 v2, v7, v10
-; GFX11TRUE16-NEXT: v_min_f32_e32 v9, v14, v13
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v8
-; GFX11TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_min_f32_e32 v5, v12, v11
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11TRUE16-NEXT: v_dual_min_f32 v0, v0, v4 :: v_dual_min_f32 v9, v14, v13
; GFX11TRUE16-NEXT: v_bfe_u32 v4, v5, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11TRUE16-NEXT: v_bfe_u32 v13, v9, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v5, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v5
; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v13, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v12, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v13, v9, 0x7fff
; GFX11TRUE16-NEXT: v_bfe_u32 v13, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.h
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v14, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11TRUE16-NEXT: v_add3_u32 v7, v13, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v11, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v12, v13, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v10, v11, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v5
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v12, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v4, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v8.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v12, v13, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_minnum_v8bf16:
@@ -22236,158 +22132,153 @@ define <16 x bfloat> @v_minnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX11TRUE16-LABEL: v_minnum_v16bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v14
; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v7
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_min_f32 v18, v19, v18 :: v_dual_and_b32 v23, 0xffff0000, v9
-; GFX11TRUE16-NEXT: v_dual_min_f32 v16, v17, v16 :: v_dual_lshlrev_b32 v15, 16, v15
-; GFX11TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v18
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_min_f32 v18, v19, v18 :: v_dual_and_b32 v21, 0xffff0000, v13
+; GFX11TRUE16-NEXT: v_min_f32_e32 v16, v17, v16
+; GFX11TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v5
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v16
+; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v18
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v16, 16, 1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11TRUE16-NEXT: v_dual_min_f32 v6, v6, v14 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_min_f32 v7, v7, v15 :: v_dual_and_b32 v14, 0xffff0000, v13
-; GFX11TRUE16-NEXT: v_bfe_u32 v15, v16, 16, 1
-; GFX11TRUE16-NEXT: v_min_f32_e32 v1, v1, v9
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v17, v7, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v7
-; GFX11TRUE16-NEXT: v_add3_u32 v15, v15, v16, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_dual_min_f32 v14, v6, v14 :: v_dual_min_f32 v15, v7, v15
+; GFX11TRUE16-NEXT: v_add3_u32 v7, v17, v16, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v16
; GFX11TRUE16-NEXT: v_bfe_u32 v16, v18, 16, 1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v7, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v15, v20, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v15, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v17, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v19, v15, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v15
; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v18, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v20, v6, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v17, v21, vcc_lo
-; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v5
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v17, v19, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v4
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_min_f32 v14, v17, v14 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v6
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v7.h
-; GFX11TRUE16-NEXT: v_min_f32_e32 v5, v5, v13
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v16, v19, vcc_lo
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v20, v6, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v18, v14, 16, 1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v12
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v5, 16, 1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v7, v15
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v16, v17, vcc_lo
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v18, v14, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v14
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v15.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v16, v20, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v16, v14, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v14
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11TRUE16-NEXT: v_add3_u32 v19, v19, v5, 0x7fff
-; GFX11TRUE16-NEXT: v_min_f32_e32 v4, v4, v12
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v14, v16, v17, vcc_lo
-; GFX11TRUE16-NEXT: v_min_f32_e32 v18, v21, v20
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v5
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11TRUE16-NEXT: v_bfe_u32 v17, v4, 16, 1
-; GFX11TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v6, v13
-; GFX11TRUE16-NEXT: v_bfe_u32 v21, v18, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v18
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v5, v19, v20 :: v_dual_and_b32 v20, 0xffff0000, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v14, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v14, v16, v20 :: v_dual_lshlrev_b32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v12
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v14.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_dual_min_f32 v12, v4, v12 :: v_dual_min_f32 v13, v5, v13
+; GFX11TRUE16-NEXT: v_dual_min_f32 v18, v19, v18 :: v_dual_min_f32 v17, v22, v21
+; GFX11TRUE16-NEXT: v_bfe_u32 v22, v12, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v13, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v20, v18, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v21, v17, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v17
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v20, v18, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_add3_u32 v5, v21, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v13
+; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v18
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v5, v5, v16 :: v_dual_and_b32 v20, 0xffff0000, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v16, v19, v13, 0x7fff
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v16, v17, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v12, v21, v18, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v22, v12, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v12
+; GFX11TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v21, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v11
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v12, v16, vcc_lo
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v17, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v4
-; GFX11TRUE16-NEXT: v_dual_min_f32 v18, v20, v19 :: v_dual_and_b32 v19, 0xffff0000, v10
-; GFX11TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v4, v16, v17 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v13.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v17, v18, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_min_f32_e32 v16, v20, v19
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v18
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_min_f32 v2, v2, v10 :: v_dual_min_f32 v3, v3, v11
-; GFX11TRUE16-NEXT: v_bfe_u32 v11, v18, 16, 1
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v12.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_bfe_u32 v19, v16, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v16
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v22, v2, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v17, v3, 16, 1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v19, v19, v16, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v11, v11, v18, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v17, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v3
-; GFX11TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v5, v14
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v12
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v10, v17, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v22, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v19, v16, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v10
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_dual_min_f32 v18, v21, v20 :: v_dual_lshlrev_b32 v11, 16, v11
+; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v9
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_bfe_u32 v20, v18, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_min_f32_e32 v1, v1, v9
+; GFX11TRUE16-NEXT: v_dual_min_f32 v10, v2, v10 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_min_f32_e32 v11, v3, v11
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v16
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v11, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v11
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v20, v18, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v19, v11, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v18
+; GFX11TRUE16-NEXT: v_bfe_u32 v20, v10, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v2, v16, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11TRUE16-NEXT: v_add3_u32 v16, v20, v10, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v10
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v0
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v19, v21, vcc_lo
-; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v2
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v8
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v2, v17, v19 :: v_dual_lshlrev_b32 v19, 16, v8
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
; GFX11TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v16, v19, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v11.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_min_f32_e32 v9, v20, v19
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v16, v18, vcc_lo
; GFX11TRUE16-NEXT: v_bfe_u32 v16, v1, 16, 1
-; GFX11TRUE16-NEXT: v_min_f32_e32 v9, v22, v21
-; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_min_f32_e32 v0, v0, v8
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_dual_min_f32 v17, v22, v21 :: v_dual_min_f32 v0, v0, v8
; GFX11TRUE16-NEXT: v_bfe_u32 v8, v9, 16, 1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v16, v22, vcc_lo
-; GFX11TRUE16-NEXT: v_min_f32_e32 v17, v24, v23
+; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_bfe_u32 v21, v17, 16, 1
; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v17
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v20, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v23, v17, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v17
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v24, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v21, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v21, v0, 16, 1
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v22, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: v_add3_u32 v19, v23, v17, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v23, v0, 16, 1
-; GFX11TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v10
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v9, v19, v21, vcc_lo
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v23, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v20, v21, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v18, v19, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v9
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v16, v22, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v11, v20, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v8, v0
-; GFX11TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v11
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v16.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v20, v21, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_minnum_v16bf16:
@@ -24090,303 +23981,274 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: scratch_load_b32 v32, off, s32
-; GFX11TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v21
-; GFX11TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v5
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v26
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; GFX11TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v19
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; GFX11TRUE16-NEXT: v_min_f32_e32 v5, v5, v21
-; GFX11TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v18
+; GFX11TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v20
+; GFX11TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v22
+; GFX11TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v6
+; GFX11TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v11
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_min_f32 v20, v4, v20 :: v_dual_lshlrev_b32 v11, 16, v11
+; GFX11TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v7
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX11TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v17
; GFX11TRUE16-NEXT: v_and_b32_e32 v84, 0xffff0000, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX11TRUE16-NEXT: v_bfe_u32 v103, v5, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v112, 0x400000, v5
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_and_b32_e32 v85, 0xffff0000, v16
-; GFX11TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v24
-; GFX11TRUE16-NEXT: v_add3_u32 v103, v103, v5, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v3
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v9
-; GFX11TRUE16-NEXT: v_dual_min_f32 v1, v1, v17 :: v_dual_lshlrev_b32 v24, 16, v24
-; GFX11TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v7
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_min_f32_e32 v3, v3, v19
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v22
-; GFX11TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v11
-; GFX11TRUE16-NEXT: v_bfe_u32 v119, v3, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v128, 0x400000, v3
-; GFX11TRUE16-NEXT: v_bfe_u32 v135, v1, 16, 1
-; GFX11TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v25
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11TRUE16-NEXT: v_add3_u32 v119, v119, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v8
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11TRUE16-NEXT: v_or_b32_e32 v144, 0x400000, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v135, v135, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_dual_min_f32 v19, v82, v81 :: v_dual_lshlrev_b32 v18, 16, v18
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX11TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v129, v19, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v130, 0x400000, v19
-; GFX11TRUE16-NEXT: v_min_f32_e32 v2, v2, v18
-; GFX11TRUE16-NEXT: v_dual_min_f32 v18, v84, v83 :: v_dual_min_f32 v9, v9, v25
-; GFX11TRUE16-NEXT: v_add3_u32 v129, v129, v19, 0x7fff
; GFX11TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v131, v2, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v132, 0x400000, v2
-; GFX11TRUE16-NEXT: v_min_f32_e32 v17, v86, v85
-; GFX11TRUE16-NEXT: v_dual_min_f32 v8, v8, v24 :: v_dual_and_b32 v39, 0xffff0000, v27
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v131, v131, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v133, v18, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v145, v17, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v146, 0x400000, v17
-; GFX11TRUE16-NEXT: v_bfe_u32 v83, v8, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v84, 0x400000, v8
-; GFX11TRUE16-NEXT: v_or_b32_e32 v134, 0x400000, v18
-; GFX11TRUE16-NEXT: v_add3_u32 v145, v145, v17, 0x7fff
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v23
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX11TRUE16-NEXT: v_bfe_u32 v115, v20, 16, 1
; GFX11TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v10
-; GFX11TRUE16-NEXT: v_add3_u32 v83, v83, v8, 0x7fff
-; GFX11TRUE16-NEXT: v_min_f32_e32 v0, v0, v16
-; GFX11TRUE16-NEXT: v_dual_min_f32 v24, v64, v55 :: v_dual_and_b32 v37, 0xffff0000, v28
-; GFX11TRUE16-NEXT: v_min_f32_e32 v7, v7, v23
-; GFX11TRUE16-NEXT: v_dual_min_f32 v23, v66, v65 :: v_dual_lshlrev_b32 v28, 16, v28
-; GFX11TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v29
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v85, v24, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v86, 0x400000, v24
-; GFX11TRUE16-NEXT: v_bfe_u32 v97, v23, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v98, 0x400000, v23
-; GFX11TRUE16-NEXT: v_bfe_u32 v87, v7, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v85, v85, v24, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v20
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11TRUE16-NEXT: v_add3_u32 v97, v97, v23, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11TRUE16-NEXT: v_or_b32_e32 v96, 0x400000, v7
-; GFX11TRUE16-NEXT: v_add3_u32 v87, v87, v7, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11TRUE16-NEXT: v_min_f32_e32 v4, v4, v20
-; GFX11TRUE16-NEXT: v_min_f32_e32 v20, v80, v71
-; GFX11TRUE16-NEXT: v_bfe_u32 v71, v9, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v80, 0x400000, v9
-; GFX11TRUE16-NEXT: v_dual_min_f32 v21, v70, v69 :: v_dual_lshlrev_b32 v10, 16, v10
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v71, v71, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX11TRUE16-NEXT: v_dual_min_f32 v10, v10, v26 :: v_dual_lshlrev_b32 v27, 16, v27
-; GFX11TRUE16-NEXT: v_dual_min_f32 v26, v52, v51 :: v_dual_min_f32 v25, v54, v53
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_min_f32_e32 v6, v6, v22
-; GFX11TRUE16-NEXT: v_dual_min_f32 v11, v11, v27 :: v_dual_and_b32 v36, 0xffff0000, v13
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v24
+; GFX11TRUE16-NEXT: v_dual_min_f32 v1, v1, v17 :: v_dual_lshlrev_b32 v24, 16, v24
+; GFX11TRUE16-NEXT: v_dual_min_f32 v23, v7, v23 :: v_dual_and_b32 v82, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_min_f32_e32 v22, v6, v22
+; GFX11TRUE16-NEXT: v_min_f32_e32 v17, v86, v85
+; GFX11TRUE16-NEXT: v_or_b32_e32 v116, 0x400000, v20
+; GFX11TRUE16-NEXT: v_add3_u32 v115, v115, v20, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v18
+; GFX11TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v8
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v27
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX11TRUE16-NEXT: v_bfe_u32 v87, v23, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v99, v22, 16, 1
+; GFX11TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v21
+; GFX11TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v5
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v19
+; GFX11TRUE16-NEXT: v_or_b32_e32 v96, 0x400000, v23
+; GFX11TRUE16-NEXT: v_or_b32_e32 v100, 0x400000, v22
+; GFX11TRUE16-NEXT: v_add3_u32 v87, v87, v23, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v3
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v99, v99, v22, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX11TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v26
+; GFX11TRUE16-NEXT: v_dual_min_f32 v21, v5, v21 :: v_dual_lshlrev_b32 v26, 16, v26
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_min_f32 v19, v3, v19 :: v_dual_and_b32 v52, 0xffff0000, v9
+; GFX11TRUE16-NEXT: v_dual_min_f32 v27, v11, v27 :: v_dual_and_b32 v36, 0xffff0000, v13
+; GFX11TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v25
+; GFX11TRUE16-NEXT: v_dual_min_f32 v26, v10, v26 :: v_dual_lshlrev_b32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_dual_min_f32 v24, v8, v24 :: v_dual_and_b32 v35, 0xffff0000, v29
; GFX11TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v30
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
; GFX11TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v14
-; GFX11TRUE16-NEXT: v_min_f32_e32 v22, v68, v67
-; GFX11TRUE16-NEXT: v_dual_min_f32 v27, v50, v49 :: v_dual_and_b32 v38, 0xffff0000, v12
+; GFX11TRUE16-NEXT: v_bfe_u32 v103, v21, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v119, v19, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_min_f32 v29, v13, v29 :: v_dual_and_b32 v38, 0xffff0000, v12
+; GFX11TRUE16-NEXT: v_dual_min_f32 v33, v34, v33 :: v_dual_lshlrev_b32 v30, 16, v30
+; GFX11TRUE16-NEXT: v_min_f32_e32 v34, v36, v35
+; GFX11TRUE16-NEXT: v_min_f32_e32 v36, v48, v39
+; GFX11TRUE16-NEXT: v_min_f32_e32 v0, v0, v16
+; GFX11TRUE16-NEXT: v_dual_min_f32 v48, v64, v55 :: v_dual_and_b32 v37, 0xffff0000, v28
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11TRUE16-NEXT: v_dual_min_f32 v13, v13, v29 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_min_f32 v29, v38, v37 :: v_dual_lshlrev_b32 v30, 16, v30
-; GFX11TRUE16-NEXT: v_and_b32_e32 v31, 0xffff0000, v15
-; GFX11TRUE16-NEXT: v_dual_min_f32 v12, v12, v28 :: v_dual_lshlrev_b32 v15, 16, v15
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_min_f32_e32 v14, v14, v30
-; GFX11TRUE16-NEXT: v_min_f32_e32 v28, v48, v39
-; GFX11TRUE16-NEXT: v_dual_min_f32 v30, v36, v35 :: v_dual_min_f32 v33, v34, v33
-; GFX11TRUE16-NEXT: v_bfe_u32 v39, v13, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v35, v14, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v14
+; GFX11TRUE16-NEXT: v_or_b32_e32 v112, 0x400000, v21
+; GFX11TRUE16-NEXT: v_or_b32_e32 v128, 0x400000, v19
+; GFX11TRUE16-NEXT: v_add3_u32 v103, v103, v21, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX11TRUE16-NEXT: v_add3_u32 v119, v119, v19, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX11TRUE16-NEXT: v_min_f32_e32 v35, v38, v37
+; GFX11TRUE16-NEXT: v_dual_min_f32 v30, v14, v30 :: v_dual_and_b32 v31, 0xffff0000, v15
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v37, v30, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v16, v33, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v34, 0x400000, v33
+; GFX11TRUE16-NEXT: v_min_f32_e32 v18, v2, v18
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v33, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v33
+; GFX11TRUE16-NEXT: v_bfe_u32 v14, v27, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v30, 16, 1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX11TRUE16-NEXT: v_add3_u32 v35, v35, v14, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v30
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v33, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v37, v37, v30, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v48, 0x400000, v13
-; GFX11TRUE16-NEXT: v_bfe_u32 v49, v29, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v39, v39, v13, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v34, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11TRUE16-NEXT: v_or_b32_e32 v50, 0x400000, v29
-; GFX11TRUE16-NEXT: v_bfe_u32 v51, v12, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v49, v49, v29, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v52, 0x400000, v12
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v14, v35, v36, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v33, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v30
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v34, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v30, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v150, v14, v27, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v14, v2, v3, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11TRUE16-NEXT: v_bfe_u32 v53, v28, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v51, v51, v12, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v54, 0x400000, v28
-; GFX11TRUE16-NEXT: v_bfe_u32 v55, v11, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v30, v37, v38, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11TRUE16-NEXT: v_add3_u32 v53, v53, v28, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v64, 0x400000, v11
-; GFX11TRUE16-NEXT: v_bfe_u32 v65, v27, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v55, v55, v11, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v39, v48, vcc_lo
+; GFX11TRUE16-NEXT: v_dual_min_f32 v39, v54, v53 :: v_dual_lshlrev_b32 v12, 16, v12
+; GFX11TRUE16-NEXT: v_dual_min_f32 v54, v84, v83 :: v_dual_lshlrev_b32 v9, 16, v9
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v34
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v29, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v13, v36, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v34, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v30, v4, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
+; GFX11TRUE16-NEXT: v_min_f32_e32 v25, v9, v25
+; GFX11TRUE16-NEXT: v_dual_min_f32 v28, v12, v28 :: v_dual_lshlrev_b32 v15, 16, v15
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v29
+; GFX11TRUE16-NEXT: v_bfe_u32 v10, v35, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v29, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v149, v13, v36, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v6, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11TRUE16-NEXT: v_or_b32_e32 v66, 0x400000, v27
-; GFX11TRUE16-NEXT: v_bfe_u32 v67, v10, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v65, v65, v27, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v68, 0x400000, v10
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v29, v49, v50, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11TRUE16-NEXT: v_bfe_u32 v69, v26, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v67, v67, v10, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v70, 0x400000, v26
-; GFX11TRUE16-NEXT: v_bfe_u32 v81, v25, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v51, v52, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v35
+; GFX11TRUE16-NEXT: v_bfe_u32 v12, v28, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v10, v35, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v28
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v29, v8, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
+; GFX11TRUE16-NEXT: v_add3_u32 v148, v12, v28, 0x7fff
+; GFX11TRUE16-NEXT: v_min_f32_e32 v37, v50, v49
+; GFX11TRUE16-NEXT: v_or_b32_e32 v55, 0x400000, v36
+; GFX11TRUE16-NEXT: v_or_b32_e32 v64, 0x400000, v27
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v10, v11, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11TRUE16-NEXT: v_add3_u32 v69, v69, v26, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v82, 0x400000, v25
-; GFX11TRUE16-NEXT: v_add3_u32 v81, v81, v25, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v99, v6, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v28, v53, v54, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11TRUE16-NEXT: v_or_b32_e32 v100, 0x400000, v6
-; GFX11TRUE16-NEXT: v_bfe_u32 v101, v22, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v99, v99, v6, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v102, 0x400000, v22
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v55, v64, vcc_lo
+; GFX11TRUE16-NEXT: v_min_f32_e32 v49, v66, v65
+; GFX11TRUE16-NEXT: v_bfe_u32 v65, v37, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v66, 0x400000, v37
+; GFX11TRUE16-NEXT: v_min_f32_e32 v38, v52, v51
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v148, v16, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
+; GFX11TRUE16-NEXT: v_add3_u32 v65, v65, v37, 0x7fff
+; GFX11TRUE16-NEXT: v_min_f32_e32 v51, v70, v69
+; GFX11TRUE16-NEXT: v_bfe_u32 v69, v38, 16, 1
+; GFX11TRUE16-NEXT: v_min_f32_e32 v52, v80, v71
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v149, v55, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11TRUE16-NEXT: v_add3_u32 v101, v101, v22, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v113, v21, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v114, 0x400000, v21
-; GFX11TRUE16-NEXT: v_bfe_u32 v115, v4, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v27, v65, v66, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11TRUE16-NEXT: v_add3_u32 v113, v113, v21, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v116, 0x400000, v4
-; GFX11TRUE16-NEXT: v_bfe_u32 v117, v20, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v115, v115, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v67, v68, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v70, 0x400000, v38
+; GFX11TRUE16-NEXT: v_bfe_u32 v71, v25, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v69, v69, v38, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v80, 0x400000, v25
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v27, v150, v64, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
+; GFX11TRUE16-NEXT: v_add3_u32 v71, v71, v25, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v83, v24, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v84, 0x400000, v24
+; GFX11TRUE16-NEXT: v_bfe_u32 v85, v48, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v65, v66, vcc_lo
+; GFX11TRUE16-NEXT: v_min_f32_e32 v50, v68, v67
+; GFX11TRUE16-NEXT: v_bfe_u32 v67, v26, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v68, 0x400000, v26
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11TRUE16-NEXT: v_or_b32_e32 v118, 0x400000, v20
-; GFX11TRUE16-NEXT: v_add3_u32 v117, v117, v20, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v83, v83, v24, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v86, 0x400000, v48
+; GFX11TRUE16-NEXT: v_add3_u32 v67, v67, v26, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v85, v85, v48, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v97, v49, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v98, 0x400000, v49
+; GFX11TRUE16-NEXT: v_bfe_u32 v101, v50, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v26, v67, v68, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
+; GFX11TRUE16-NEXT: v_add3_u32 v97, v97, v49, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v102, 0x400000, v50
+; GFX11TRUE16-NEXT: v_add3_u32 v101, v101, v50, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v113, v51, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v9, v69, v70, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11TRUE16-NEXT: v_or_b32_e32 v114, 0x400000, v51
+; GFX11TRUE16-NEXT: v_bfe_u32 v117, v52, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v113, v113, v51, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v118, 0x400000, v52
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v25, v71, v80, vcc_lo
+; GFX11TRUE16-NEXT: v_min_f32_e32 v53, v82, v81
+; GFX11TRUE16-NEXT: v_bfe_u32 v81, v39, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v82, 0x400000, v39
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v39, v39
+; GFX11TRUE16-NEXT: v_add3_u32 v117, v117, v52, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v129, v53, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v81, v81, v39, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v130, 0x400000, v53
+; GFX11TRUE16-NEXT: v_bfe_u32 v131, v18, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v132, 0x400000, v18
+; GFX11TRUE16-NEXT: v_add3_u32 v129, v129, v53, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v81, v82, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11TRUE16-NEXT: v_bfe_u32 v135, v1, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v131, v131, v18, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v144, 0x400000, v1
; GFX11TRUE16-NEXT: v_bfe_u32 v147, v0, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v24, v83, v84, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
+; GFX11TRUE16-NEXT: v_add3_u32 v135, v135, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v133, v54, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v26, v69, v70, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11TRUE16-NEXT: v_add3_u32 v133, v133, v18, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v147, v147, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v10.l, v10.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v11.l, v11.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v9, v71, v80, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v12.l, v12.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v13.l, v13.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v14.l, v14.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v9.l, v9.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v25, v81, v82, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v10, v27
-; GFX11TRUE16-NEXT: v_bfi_b32 v11, 0xffff, v11, v28
-; GFX11TRUE16-NEXT: v_bfi_b32 v9, 0xffff, v9, v26
-; GFX11TRUE16-NEXT: v_bfi_b32 v12, 0xffff, v12, v29
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v83, v84, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11TRUE16-NEXT: v_bfi_b32 v13, 0xffff, v13, v30
-; GFX11TRUE16-NEXT: v_bfi_b32 v14, 0xffff, v14, v16
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v24, v85, v86, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11TRUE16-NEXT: v_bfi_b32 v8, 0xffff, v8, v25
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v87, v96, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v85, v86, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v7.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v23, v97, v98, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v7, v24
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v99, v100, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v134, 0x400000, v54
+; GFX11TRUE16-NEXT: v_bfe_u32 v145, v17, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v133, v133, v54, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v146, 0x400000, v17
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v23, v87, v96, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v49, v49
+; GFX11TRUE16-NEXT: v_add3_u32 v145, v145, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v12.l, v16.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v8.l, v24.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v23.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v97, v98, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v22, v101, v102, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v6, v23
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v103, v112, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v9.l, v25.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v10.l, v26.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v11.l, v27.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v13.l, v29.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v22, v99, v100, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v50, v50
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v14.l, v30.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v22.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v101, v102, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v21, v113, v114, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v5, v22
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v115, v116, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v21, v103, v112, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v51, v51
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v21.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v113, v114, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v20, v117, v118, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v21
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v119, v128, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v20, v115, v116, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v52, v52
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v20.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v117, v118, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v19, v129, v130, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v20
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v131, v132, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v19, v119, v128, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v53, v53
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v19.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v129, v130, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v131, v132, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v135, v144, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v18.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v28, v135, v144, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v19
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v147, v33, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v133, v134, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v33, v147, v33, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v54, v54
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v133, v134, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v18
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v17, v145, v146, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v28.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v145, v146, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v33.h
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v32
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v17
-; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v32
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_min_f32_e32 v15, v15, v33
-; GFX11TRUE16-NEXT: v_min_f32_e32 v17, v31, v17
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v32
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_dual_min_f32 v15, v15, v17 :: v_dual_and_b32 v18, 0xffff0000, v32
+; GFX11TRUE16-NEXT: v_min_f32_e32 v17, v31, v18
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_bfe_u32 v18, v15, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v17, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v15
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v17, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v17
+; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v17
; GFX11TRUE16-NEXT: v_add3_u32 v18, v18, v15, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_add3_u32 v19, v19, v17, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v18, v20, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v18, v20, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v15.l, v15.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v17, v19, v21, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v15, 0xffff, v15, v17
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v19, v16, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v15.l, v18.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_minnum_v32bf16:
@@ -24671,6 +24533,7 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
ret <32 x bfloat> %op
}
+
declare bfloat @llvm.maxnum.bf16(bfloat, bfloat)
declare <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat>, <2 x bfloat>)
declare <3 x bfloat> @llvm.maxnum.v3bf16(<3 x bfloat>, <3 x bfloat>)
@@ -24930,13 +24793,12 @@ define <2 x bfloat> @v_maxnum_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) {
; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_maxnum_v2bf16:
@@ -25134,37 +24996,36 @@ define <3 x bfloat> @v_maxnum_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) {
; GFX11TRUE16-LABEL: v_maxnum_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_max_f32 v1, v1, v3 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_max_f32_e32 v0, v0, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_dual_max_f32 v0, v0, v2 :: v_dual_lshlrev_b32 v3, 16, v3
; GFX11TRUE16-NEXT: v_max_f32_e32 v4, v5, v4
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v5, v6, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_max_f32_e32 v1, v1, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v4, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_maxnum_v3bf16:
@@ -25434,21 +25295,17 @@ define <4 x bfloat> @v_maxnum_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) {
; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v4
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v9, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v9, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v10, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v3, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_maxnum_v4bf16:
@@ -25876,81 +25733,77 @@ define <8 x bfloat> @v_maxnum_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) {
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
+; GFX11TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
+; GFX11TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v3
-; GFX11TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_dual_max_f32 v8, v9, v8 :: v_dual_lshlrev_b32 v1, 16, v1
; GFX11TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_max_f32_e32 v7, v3, v7
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_max_f32_e32 v1, v1, v5
; GFX11TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_max_f32 v2, v2, v6 :: v_dual_max_f32 v3, v3, v7
-; GFX11TRUE16-NEXT: v_max_f32_e32 v7, v10, v9
-; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v8
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_max_f32_e32 v9, v10, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v8
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
; GFX11TRUE16-NEXT: v_add3_u32 v10, v11, v8, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v11, v3, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v12, v7, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v10, v9, vcc_lo
-; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v11, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v12, v7, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX11TRUE16-NEXT: v_bfe_u32 v12, v2, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_bfe_u32 v11, v7, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v12, v9, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v7
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc_lo
+; GFX11TRUE16-NEXT: v_max_f32_e32 v6, v2, v6
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v11, v7, 0x7fff
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v12, v9, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
+; GFX11TRUE16-NEXT: v_bfe_u32 v12, v6, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v2, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add3_u32 v7, v12, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v12, v6, 0x7fff
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v0
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v6, v10, v11 :: v_dual_lshlrev_b32 v11, 16, v4
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v10, v11, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v4
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_max_f32 v0, v0, v4 :: v_dual_max_f32 v1, v1, v5
-; GFX11TRUE16-NEXT: v_dual_max_f32 v5, v12, v11 :: v_dual_cndmask_b32 v2, v7, v10
-; GFX11TRUE16-NEXT: v_max_f32_e32 v9, v14, v13
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v8
-; GFX11TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_max_f32_e32 v5, v12, v11
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11TRUE16-NEXT: v_dual_max_f32 v0, v0, v4 :: v_dual_max_f32 v9, v14, v13
; GFX11TRUE16-NEXT: v_bfe_u32 v4, v5, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11TRUE16-NEXT: v_bfe_u32 v13, v9, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v5, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v5
; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v13, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v12, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v13, v9, 0x7fff
; GFX11TRUE16-NEXT: v_bfe_u32 v13, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.h
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v14, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11TRUE16-NEXT: v_add3_u32 v7, v13, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v11, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v12, v13, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v10, v11, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v5
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v12, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v4, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v8.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v12, v13, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_maxnum_v8bf16:
@@ -26760,158 +26613,153 @@ define <16 x bfloat> @v_maxnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) {
; GFX11TRUE16-LABEL: v_maxnum_v16bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v14
; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v7
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_max_f32 v18, v19, v18 :: v_dual_and_b32 v23, 0xffff0000, v9
-; GFX11TRUE16-NEXT: v_dual_max_f32 v16, v17, v16 :: v_dual_lshlrev_b32 v15, 16, v15
-; GFX11TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v18
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_max_f32 v18, v19, v18 :: v_dual_and_b32 v21, 0xffff0000, v13
+; GFX11TRUE16-NEXT: v_max_f32_e32 v16, v17, v16
+; GFX11TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v5
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v16
+; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v18
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v16, 16, 1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11TRUE16-NEXT: v_dual_max_f32 v6, v6, v14 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_max_f32 v7, v7, v15 :: v_dual_and_b32 v14, 0xffff0000, v13
-; GFX11TRUE16-NEXT: v_bfe_u32 v15, v16, 16, 1
-; GFX11TRUE16-NEXT: v_max_f32_e32 v1, v1, v9
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v17, v7, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v7
-; GFX11TRUE16-NEXT: v_add3_u32 v15, v15, v16, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_dual_max_f32 v14, v6, v14 :: v_dual_max_f32 v15, v7, v15
+; GFX11TRUE16-NEXT: v_add3_u32 v7, v17, v16, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v16
; GFX11TRUE16-NEXT: v_bfe_u32 v16, v18, 16, 1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v7, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v15, v20, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v15, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v17, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v19, v15, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v15
; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v18, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v20, v6, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v17, v21, vcc_lo
-; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v5
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v17, v19, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v4
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_max_f32 v14, v17, v14 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v6
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v7.h
-; GFX11TRUE16-NEXT: v_max_f32_e32 v5, v5, v13
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v16, v19, vcc_lo
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v20, v6, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v18, v14, 16, 1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v12
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v5, 16, 1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v7, v15
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v16, v17, vcc_lo
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v18, v14, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v14
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v15.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v16, v20, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v16, v14, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v14
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11TRUE16-NEXT: v_add3_u32 v19, v19, v5, 0x7fff
-; GFX11TRUE16-NEXT: v_max_f32_e32 v4, v4, v12
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v14, v16, v17, vcc_lo
-; GFX11TRUE16-NEXT: v_max_f32_e32 v18, v21, v20
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v5
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11TRUE16-NEXT: v_bfe_u32 v17, v4, 16, 1
-; GFX11TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v6, v13
-; GFX11TRUE16-NEXT: v_bfe_u32 v21, v18, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v18
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v5, v19, v20 :: v_dual_and_b32 v20, 0xffff0000, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v14, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v14, v16, v20 :: v_dual_lshlrev_b32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v12
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v14.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_dual_max_f32 v12, v4, v12 :: v_dual_max_f32 v13, v5, v13
+; GFX11TRUE16-NEXT: v_dual_max_f32 v18, v19, v18 :: v_dual_max_f32 v17, v22, v21
+; GFX11TRUE16-NEXT: v_bfe_u32 v22, v12, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v13, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v20, v18, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v21, v17, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v17
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v20, v18, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_add3_u32 v5, v21, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v13
+; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v18
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v5, v5, v16 :: v_dual_and_b32 v20, 0xffff0000, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v16, v19, v13, 0x7fff
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v16, v17, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v12, v21, v18, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v22, v12, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v12
+; GFX11TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v21, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v11
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v12, v16, vcc_lo
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v17, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v4
-; GFX11TRUE16-NEXT: v_dual_max_f32 v18, v20, v19 :: v_dual_and_b32 v19, 0xffff0000, v10
-; GFX11TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v4, v16, v17 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v13.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v17, v18, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_max_f32_e32 v16, v20, v19
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v18
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_max_f32 v2, v2, v10 :: v_dual_max_f32 v3, v3, v11
-; GFX11TRUE16-NEXT: v_bfe_u32 v11, v18, 16, 1
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v12.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_bfe_u32 v19, v16, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v16
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v22, v2, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v17, v3, 16, 1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v19, v19, v16, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v11, v11, v18, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v17, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v3
-; GFX11TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v5, v14
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v12
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v10, v17, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v22, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v19, v16, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v10
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_dual_max_f32 v18, v21, v20 :: v_dual_lshlrev_b32 v11, 16, v11
+; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v9
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_bfe_u32 v20, v18, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_max_f32_e32 v1, v1, v9
+; GFX11TRUE16-NEXT: v_dual_max_f32 v10, v2, v10 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_max_f32_e32 v11, v3, v11
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v16
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v11, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v11
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v20, v18, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v19, v11, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v18
+; GFX11TRUE16-NEXT: v_bfe_u32 v20, v10, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v2, v16, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11TRUE16-NEXT: v_add3_u32 v16, v20, v10, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v10
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v0
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v19, v21, vcc_lo
-; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v2
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v8
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v2, v17, v19 :: v_dual_lshlrev_b32 v19, 16, v8
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
; GFX11TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v16, v19, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v11.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_max_f32_e32 v9, v20, v19
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v16, v18, vcc_lo
; GFX11TRUE16-NEXT: v_bfe_u32 v16, v1, 16, 1
-; GFX11TRUE16-NEXT: v_max_f32_e32 v9, v22, v21
-; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_max_f32_e32 v0, v0, v8
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_dual_max_f32 v17, v22, v21 :: v_dual_max_f32 v0, v0, v8
; GFX11TRUE16-NEXT: v_bfe_u32 v8, v9, 16, 1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v16, v22, vcc_lo
-; GFX11TRUE16-NEXT: v_max_f32_e32 v17, v24, v23
+; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_bfe_u32 v21, v17, 16, 1
; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v17
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v20, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v23, v17, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v17
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v24, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v21, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v21, v0, 16, 1
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v22, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: v_add3_u32 v19, v23, v17, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v23, v0, 16, 1
-; GFX11TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v10
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v9, v19, v21, vcc_lo
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v23, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v20, v21, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v18, v19, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v9
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v16, v22, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v11, v20, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v8, v0
-; GFX11TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v11
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v16.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v20, v21, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_maxnum_v16bf16:
@@ -28614,303 +28462,274 @@ define <32 x bfloat> @v_maxnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: scratch_load_b32 v32, off, s32
-; GFX11TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v21
-; GFX11TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v5
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v26
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; GFX11TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v19
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; GFX11TRUE16-NEXT: v_max_f32_e32 v5, v5, v21
-; GFX11TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v18
+; GFX11TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v20
+; GFX11TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v22
+; GFX11TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v6
+; GFX11TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v11
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_max_f32 v20, v4, v20 :: v_dual_lshlrev_b32 v11, 16, v11
+; GFX11TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v7
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX11TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v17
; GFX11TRUE16-NEXT: v_and_b32_e32 v84, 0xffff0000, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX11TRUE16-NEXT: v_bfe_u32 v103, v5, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v112, 0x400000, v5
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_and_b32_e32 v85, 0xffff0000, v16
-; GFX11TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v24
-; GFX11TRUE16-NEXT: v_add3_u32 v103, v103, v5, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v3
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v9
-; GFX11TRUE16-NEXT: v_dual_max_f32 v1, v1, v17 :: v_dual_lshlrev_b32 v24, 16, v24
-; GFX11TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v7
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_max_f32_e32 v3, v3, v19
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v22
-; GFX11TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v11
-; GFX11TRUE16-NEXT: v_bfe_u32 v119, v3, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v128, 0x400000, v3
-; GFX11TRUE16-NEXT: v_bfe_u32 v135, v1, 16, 1
-; GFX11TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v25
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11TRUE16-NEXT: v_add3_u32 v119, v119, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v8
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11TRUE16-NEXT: v_or_b32_e32 v144, 0x400000, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v135, v135, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_dual_max_f32 v19, v82, v81 :: v_dual_lshlrev_b32 v18, 16, v18
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX11TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v129, v19, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v130, 0x400000, v19
-; GFX11TRUE16-NEXT: v_max_f32_e32 v2, v2, v18
-; GFX11TRUE16-NEXT: v_dual_max_f32 v18, v84, v83 :: v_dual_max_f32 v9, v9, v25
-; GFX11TRUE16-NEXT: v_add3_u32 v129, v129, v19, 0x7fff
; GFX11TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v131, v2, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v132, 0x400000, v2
-; GFX11TRUE16-NEXT: v_max_f32_e32 v17, v86, v85
-; GFX11TRUE16-NEXT: v_dual_max_f32 v8, v8, v24 :: v_dual_and_b32 v39, 0xffff0000, v27
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v131, v131, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v133, v18, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v145, v17, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v146, 0x400000, v17
-; GFX11TRUE16-NEXT: v_bfe_u32 v83, v8, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v84, 0x400000, v8
-; GFX11TRUE16-NEXT: v_or_b32_e32 v134, 0x400000, v18
-; GFX11TRUE16-NEXT: v_add3_u32 v145, v145, v17, 0x7fff
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v23
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
+; GFX11TRUE16-NEXT: v_bfe_u32 v115, v20, 16, 1
; GFX11TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v10
-; GFX11TRUE16-NEXT: v_add3_u32 v83, v83, v8, 0x7fff
-; GFX11TRUE16-NEXT: v_max_f32_e32 v0, v0, v16
-; GFX11TRUE16-NEXT: v_dual_max_f32 v24, v64, v55 :: v_dual_and_b32 v37, 0xffff0000, v28
-; GFX11TRUE16-NEXT: v_max_f32_e32 v7, v7, v23
-; GFX11TRUE16-NEXT: v_dual_max_f32 v23, v66, v65 :: v_dual_lshlrev_b32 v28, 16, v28
-; GFX11TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v29
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v85, v24, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v86, 0x400000, v24
-; GFX11TRUE16-NEXT: v_bfe_u32 v97, v23, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v98, 0x400000, v23
-; GFX11TRUE16-NEXT: v_bfe_u32 v87, v7, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v85, v85, v24, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v20
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11TRUE16-NEXT: v_add3_u32 v97, v97, v23, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11TRUE16-NEXT: v_or_b32_e32 v96, 0x400000, v7
-; GFX11TRUE16-NEXT: v_add3_u32 v87, v87, v7, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11TRUE16-NEXT: v_max_f32_e32 v4, v4, v20
-; GFX11TRUE16-NEXT: v_max_f32_e32 v20, v80, v71
-; GFX11TRUE16-NEXT: v_bfe_u32 v71, v9, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v80, 0x400000, v9
-; GFX11TRUE16-NEXT: v_dual_max_f32 v21, v70, v69 :: v_dual_lshlrev_b32 v10, 16, v10
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v71, v71, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX11TRUE16-NEXT: v_dual_max_f32 v10, v10, v26 :: v_dual_lshlrev_b32 v27, 16, v27
-; GFX11TRUE16-NEXT: v_dual_max_f32 v26, v52, v51 :: v_dual_max_f32 v25, v54, v53
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_max_f32_e32 v6, v6, v22
-; GFX11TRUE16-NEXT: v_dual_max_f32 v11, v11, v27 :: v_dual_and_b32 v36, 0xffff0000, v13
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v24
+; GFX11TRUE16-NEXT: v_dual_max_f32 v1, v1, v17 :: v_dual_lshlrev_b32 v24, 16, v24
+; GFX11TRUE16-NEXT: v_dual_max_f32 v23, v7, v23 :: v_dual_and_b32 v82, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_max_f32_e32 v22, v6, v22
+; GFX11TRUE16-NEXT: v_max_f32_e32 v17, v86, v85
+; GFX11TRUE16-NEXT: v_or_b32_e32 v116, 0x400000, v20
+; GFX11TRUE16-NEXT: v_add3_u32 v115, v115, v20, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v18
+; GFX11TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v8
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v27
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX11TRUE16-NEXT: v_bfe_u32 v87, v23, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v99, v22, 16, 1
+; GFX11TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v21
+; GFX11TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v5
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v19
+; GFX11TRUE16-NEXT: v_or_b32_e32 v96, 0x400000, v23
+; GFX11TRUE16-NEXT: v_or_b32_e32 v100, 0x400000, v22
+; GFX11TRUE16-NEXT: v_add3_u32 v87, v87, v23, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v3
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v99, v99, v22, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX11TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v26
+; GFX11TRUE16-NEXT: v_dual_max_f32 v21, v5, v21 :: v_dual_lshlrev_b32 v26, 16, v26
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_max_f32 v19, v3, v19 :: v_dual_and_b32 v52, 0xffff0000, v9
+; GFX11TRUE16-NEXT: v_dual_max_f32 v27, v11, v27 :: v_dual_and_b32 v36, 0xffff0000, v13
+; GFX11TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v25
+; GFX11TRUE16-NEXT: v_dual_max_f32 v26, v10, v26 :: v_dual_lshlrev_b32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_dual_max_f32 v24, v8, v24 :: v_dual_and_b32 v35, 0xffff0000, v29
; GFX11TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v30
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
; GFX11TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v14
-; GFX11TRUE16-NEXT: v_max_f32_e32 v22, v68, v67
-; GFX11TRUE16-NEXT: v_dual_max_f32 v27, v50, v49 :: v_dual_and_b32 v38, 0xffff0000, v12
+; GFX11TRUE16-NEXT: v_bfe_u32 v103, v21, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v119, v19, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_max_f32 v29, v13, v29 :: v_dual_and_b32 v38, 0xffff0000, v12
+; GFX11TRUE16-NEXT: v_dual_max_f32 v33, v34, v33 :: v_dual_lshlrev_b32 v30, 16, v30
+; GFX11TRUE16-NEXT: v_max_f32_e32 v34, v36, v35
+; GFX11TRUE16-NEXT: v_max_f32_e32 v36, v48, v39
+; GFX11TRUE16-NEXT: v_max_f32_e32 v0, v0, v16
+; GFX11TRUE16-NEXT: v_dual_max_f32 v48, v64, v55 :: v_dual_and_b32 v37, 0xffff0000, v28
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11TRUE16-NEXT: v_dual_max_f32 v13, v13, v29 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_max_f32 v29, v38, v37 :: v_dual_lshlrev_b32 v30, 16, v30
-; GFX11TRUE16-NEXT: v_and_b32_e32 v31, 0xffff0000, v15
-; GFX11TRUE16-NEXT: v_dual_max_f32 v12, v12, v28 :: v_dual_lshlrev_b32 v15, 16, v15
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_max_f32_e32 v14, v14, v30
-; GFX11TRUE16-NEXT: v_max_f32_e32 v28, v48, v39
-; GFX11TRUE16-NEXT: v_dual_max_f32 v30, v36, v35 :: v_dual_max_f32 v33, v34, v33
-; GFX11TRUE16-NEXT: v_bfe_u32 v39, v13, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v35, v14, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v14
+; GFX11TRUE16-NEXT: v_or_b32_e32 v112, 0x400000, v21
+; GFX11TRUE16-NEXT: v_or_b32_e32 v128, 0x400000, v19
+; GFX11TRUE16-NEXT: v_add3_u32 v103, v103, v21, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX11TRUE16-NEXT: v_add3_u32 v119, v119, v19, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX11TRUE16-NEXT: v_max_f32_e32 v35, v38, v37
+; GFX11TRUE16-NEXT: v_dual_max_f32 v30, v14, v30 :: v_dual_and_b32 v31, 0xffff0000, v15
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v37, v30, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v16, v33, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v34, 0x400000, v33
+; GFX11TRUE16-NEXT: v_max_f32_e32 v18, v2, v18
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v33, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v33
+; GFX11TRUE16-NEXT: v_bfe_u32 v14, v27, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v30, 16, 1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX11TRUE16-NEXT: v_add3_u32 v35, v35, v14, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v30
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v33, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v37, v37, v30, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v48, 0x400000, v13
-; GFX11TRUE16-NEXT: v_bfe_u32 v49, v29, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v39, v39, v13, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v34, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11TRUE16-NEXT: v_or_b32_e32 v50, 0x400000, v29
-; GFX11TRUE16-NEXT: v_bfe_u32 v51, v12, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v49, v49, v29, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v52, 0x400000, v12
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v14, v35, v36, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v33, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v30
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v34, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v30, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v150, v14, v27, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v14, v2, v3, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11TRUE16-NEXT: v_bfe_u32 v53, v28, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v51, v51, v12, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v54, 0x400000, v28
-; GFX11TRUE16-NEXT: v_bfe_u32 v55, v11, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v30, v37, v38, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11TRUE16-NEXT: v_add3_u32 v53, v53, v28, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v64, 0x400000, v11
-; GFX11TRUE16-NEXT: v_bfe_u32 v65, v27, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v55, v55, v11, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v39, v48, vcc_lo
+; GFX11TRUE16-NEXT: v_dual_max_f32 v39, v54, v53 :: v_dual_lshlrev_b32 v12, 16, v12
+; GFX11TRUE16-NEXT: v_dual_max_f32 v54, v84, v83 :: v_dual_lshlrev_b32 v9, 16, v9
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v34
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v29, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v13, v36, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v34, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v30, v4, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
+; GFX11TRUE16-NEXT: v_max_f32_e32 v25, v9, v25
+; GFX11TRUE16-NEXT: v_dual_max_f32 v28, v12, v28 :: v_dual_lshlrev_b32 v15, 16, v15
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v29
+; GFX11TRUE16-NEXT: v_bfe_u32 v10, v35, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v29, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v149, v13, v36, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v6, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11TRUE16-NEXT: v_or_b32_e32 v66, 0x400000, v27
-; GFX11TRUE16-NEXT: v_bfe_u32 v67, v10, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v65, v65, v27, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v68, 0x400000, v10
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v29, v49, v50, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11TRUE16-NEXT: v_bfe_u32 v69, v26, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v67, v67, v10, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v70, 0x400000, v26
-; GFX11TRUE16-NEXT: v_bfe_u32 v81, v25, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v51, v52, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v35
+; GFX11TRUE16-NEXT: v_bfe_u32 v12, v28, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v10, v35, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v28
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v29, v8, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
+; GFX11TRUE16-NEXT: v_add3_u32 v148, v12, v28, 0x7fff
+; GFX11TRUE16-NEXT: v_max_f32_e32 v37, v50, v49
+; GFX11TRUE16-NEXT: v_or_b32_e32 v55, 0x400000, v36
+; GFX11TRUE16-NEXT: v_or_b32_e32 v64, 0x400000, v27
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v10, v11, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11TRUE16-NEXT: v_add3_u32 v69, v69, v26, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v82, 0x400000, v25
-; GFX11TRUE16-NEXT: v_add3_u32 v81, v81, v25, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v99, v6, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v28, v53, v54, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11TRUE16-NEXT: v_or_b32_e32 v100, 0x400000, v6
-; GFX11TRUE16-NEXT: v_bfe_u32 v101, v22, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v99, v99, v6, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v102, 0x400000, v22
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v55, v64, vcc_lo
+; GFX11TRUE16-NEXT: v_max_f32_e32 v49, v66, v65
+; GFX11TRUE16-NEXT: v_bfe_u32 v65, v37, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v66, 0x400000, v37
+; GFX11TRUE16-NEXT: v_max_f32_e32 v38, v52, v51
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v148, v16, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
+; GFX11TRUE16-NEXT: v_add3_u32 v65, v65, v37, 0x7fff
+; GFX11TRUE16-NEXT: v_max_f32_e32 v51, v70, v69
+; GFX11TRUE16-NEXT: v_bfe_u32 v69, v38, 16, 1
+; GFX11TRUE16-NEXT: v_max_f32_e32 v52, v80, v71
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v149, v55, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11TRUE16-NEXT: v_add3_u32 v101, v101, v22, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v113, v21, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v114, 0x400000, v21
-; GFX11TRUE16-NEXT: v_bfe_u32 v115, v4, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v27, v65, v66, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11TRUE16-NEXT: v_add3_u32 v113, v113, v21, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v116, 0x400000, v4
-; GFX11TRUE16-NEXT: v_bfe_u32 v117, v20, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v115, v115, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v67, v68, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v70, 0x400000, v38
+; GFX11TRUE16-NEXT: v_bfe_u32 v71, v25, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v69, v69, v38, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v80, 0x400000, v25
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v27, v150, v64, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
+; GFX11TRUE16-NEXT: v_add3_u32 v71, v71, v25, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v83, v24, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v84, 0x400000, v24
+; GFX11TRUE16-NEXT: v_bfe_u32 v85, v48, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v65, v66, vcc_lo
+; GFX11TRUE16-NEXT: v_max_f32_e32 v50, v68, v67
+; GFX11TRUE16-NEXT: v_bfe_u32 v67, v26, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v68, 0x400000, v26
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11TRUE16-NEXT: v_or_b32_e32 v118, 0x400000, v20
-; GFX11TRUE16-NEXT: v_add3_u32 v117, v117, v20, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v83, v83, v24, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v86, 0x400000, v48
+; GFX11TRUE16-NEXT: v_add3_u32 v67, v67, v26, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v85, v85, v48, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v97, v49, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v98, 0x400000, v49
+; GFX11TRUE16-NEXT: v_bfe_u32 v101, v50, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v26, v67, v68, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
+; GFX11TRUE16-NEXT: v_add3_u32 v97, v97, v49, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v102, 0x400000, v50
+; GFX11TRUE16-NEXT: v_add3_u32 v101, v101, v50, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v113, v51, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v9, v69, v70, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11TRUE16-NEXT: v_or_b32_e32 v114, 0x400000, v51
+; GFX11TRUE16-NEXT: v_bfe_u32 v117, v52, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v113, v113, v51, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v118, 0x400000, v52
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v25, v71, v80, vcc_lo
+; GFX11TRUE16-NEXT: v_max_f32_e32 v53, v82, v81
+; GFX11TRUE16-NEXT: v_bfe_u32 v81, v39, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v82, 0x400000, v39
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v39, v39
+; GFX11TRUE16-NEXT: v_add3_u32 v117, v117, v52, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v129, v53, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v81, v81, v39, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v130, 0x400000, v53
+; GFX11TRUE16-NEXT: v_bfe_u32 v131, v18, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v132, 0x400000, v18
+; GFX11TRUE16-NEXT: v_add3_u32 v129, v129, v53, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v81, v82, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11TRUE16-NEXT: v_bfe_u32 v135, v1, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v131, v131, v18, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v144, 0x400000, v1
; GFX11TRUE16-NEXT: v_bfe_u32 v147, v0, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v24, v83, v84, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
+; GFX11TRUE16-NEXT: v_add3_u32 v135, v135, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v133, v54, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v26, v69, v70, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11TRUE16-NEXT: v_add3_u32 v133, v133, v18, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v147, v147, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v10.l, v10.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v11.l, v11.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v9, v71, v80, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v12.l, v12.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v13.l, v13.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v14.l, v14.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v9.l, v9.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v25, v81, v82, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11TRUE16-NEXT: v_bfi_b32 v10, 0xffff, v10, v27
-; GFX11TRUE16-NEXT: v_bfi_b32 v11, 0xffff, v11, v28
-; GFX11TRUE16-NEXT: v_bfi_b32 v9, 0xffff, v9, v26
-; GFX11TRUE16-NEXT: v_bfi_b32 v12, 0xffff, v12, v29
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v83, v84, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11TRUE16-NEXT: v_bfi_b32 v13, 0xffff, v13, v30
-; GFX11TRUE16-NEXT: v_bfi_b32 v14, 0xffff, v14, v16
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v24, v85, v86, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11TRUE16-NEXT: v_bfi_b32 v8, 0xffff, v8, v25
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v87, v96, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v85, v86, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v7.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v23, v97, v98, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11TRUE16-NEXT: v_bfi_b32 v7, 0xffff, v7, v24
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v99, v100, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v134, 0x400000, v54
+; GFX11TRUE16-NEXT: v_bfe_u32 v145, v17, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v133, v133, v54, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v146, 0x400000, v17
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v23, v87, v96, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v49, v49
+; GFX11TRUE16-NEXT: v_add3_u32 v145, v145, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v12.l, v16.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v8.l, v24.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v23.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v97, v98, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v22, v101, v102, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11TRUE16-NEXT: v_bfi_b32 v6, 0xffff, v6, v23
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v103, v112, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v9.l, v25.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v10.l, v26.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v11.l, v27.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v13.l, v29.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v22, v99, v100, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v50, v50
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v14.l, v30.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v22.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v101, v102, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v5.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v21, v113, v114, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v5, v22
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v115, v116, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v21, v103, v112, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v51, v51
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v21.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v113, v114, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v20, v117, v118, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v21
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v119, v128, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v20, v115, v116, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v52, v52
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v20.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v117, v118, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v19, v129, v130, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v3, v20
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v131, v132, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v19, v119, v128, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v53, v53
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v19.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v129, v130, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v131, v132, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v135, v144, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v18.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v28, v135, v144, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v19
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v147, v33, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v133, v134, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v33, v147, v33, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v54, v54
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v133, v134, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v18
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v17, v145, v146, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v28.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v145, v146, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v33.h
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v32
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v17
-; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v32
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_max_f32_e32 v15, v15, v33
-; GFX11TRUE16-NEXT: v_max_f32_e32 v17, v31, v17
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v32
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_dual_max_f32 v15, v15, v17 :: v_dual_and_b32 v18, 0xffff0000, v32
+; GFX11TRUE16-NEXT: v_max_f32_e32 v17, v31, v18
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_bfe_u32 v18, v15, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v17, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v15
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v17, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v17
+; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v17
; GFX11TRUE16-NEXT: v_add3_u32 v18, v18, v15, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_add3_u32 v19, v19, v17, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v18, v20, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v18, v20, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v15.l, v15.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v17, v19, v21, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v15, 0xffff, v15, v17
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v19, v16, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v15.l, v18.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_maxnum_v32bf16:
@@ -29672,6 +29491,7 @@ define { bfloat, i16 } @v_frexp_bf16_i16(bfloat %a) {
ret { bfloat, i16 } %op
}
+
declare bfloat @llvm.log.bf16(bfloat)
declare bfloat @llvm.log2.bf16(bfloat)
declare bfloat @llvm.log10.bf16(bfloat)
@@ -35089,9 +34909,8 @@ define <2 x bfloat> @v_sitofp_v2i16_to_v2bf16(<2 x i16> %x) {
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_sitofp_v2i16_to_v2bf16:
@@ -35245,35 +35064,33 @@ define <3 x bfloat> @v_sitofp_v3i16_to_v3bf16(<3 x i16> %x) {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_bfe_i32 v2, v0, 0, 16
-; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX11TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v5, v6, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_sitofp_v3i16_to_v3bf16:
@@ -35486,20 +35303,17 @@ define <4 x bfloat> @v_sitofp_v4i16_to_v4bf16(<4 x i16> %x) {
; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v11, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v9, v11, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v10, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v2
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v3
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v10, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_sitofp_v4i16_to_v4bf16:
@@ -35736,12 +35550,11 @@ define <2 x bfloat> @v_sitofp_v2i32_to_v2bf16(<2 x i32> %x) {
; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_sitofp_v2i32_to_v2bf16:
@@ -35887,30 +35700,28 @@ define <3 x bfloat> @v_sitofp_v3i32_to_v3bf16(<3 x i32> %x) {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
+; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v2
-; GFX11TRUE16-NEXT: v_add3_u32 v5, v6, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v3, vcc_lo
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_sitofp_v3i32_to_v3bf16:
@@ -36087,36 +35898,32 @@ define <4 x bfloat> @v_sitofp_v4i32_to_v4bf16(<4 x i32> %x) {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v3, v3
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v3, v3
+; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v4, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX11TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_bfe_u32 v1, v3, 16, 1
; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v10, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v3, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v4, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v9, v10, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v6, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v2, v3
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_sitofp_v4i32_to_v4bf16:
@@ -36608,13 +36415,12 @@ define <2 x bfloat> @v_sitofp_v2i64_to_v2bf16(<2 x i64> %x) {
; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_sitofp_v2i64_to_v2bf16:
@@ -36989,70 +36795,70 @@ define <3 x bfloat> @v_sitofp_v3i64_to_v3bf16(<3 x i64> %x) {
; GFX11TRUE16-LABEL: v_sitofp_v3i64_to_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_xor_b32_e32 v6, v0, v1
+; GFX11TRUE16-NEXT: v_xor_b32_e32 v7, v0, v1
+; GFX11TRUE16-NEXT: v_xor_b32_e32 v8, v4, v5
; GFX11TRUE16-NEXT: v_xor_b32_e32 v9, v2, v3
; GFX11TRUE16-NEXT: v_cls_i32_e32 v10, v1
-; GFX11TRUE16-NEXT: v_xor_b32_e32 v7, v4, v5
+; GFX11TRUE16-NEXT: v_cls_i32_e32 v6, v5
+; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v7, 31, v7
+; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v8, 31, v8
; GFX11TRUE16-NEXT: v_cls_i32_e32 v11, v3
-; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v6, 31, v6
; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v9, 31, v9
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v10, -1, v10
-; GFX11TRUE16-NEXT: v_cls_i32_e32 v8, v5
-; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v7, 31, v7
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v6, 32, v6
+; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v7, 32, v7
+; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v6, -1, v6
+; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v8, 32, v8
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v11, -1, v11
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v9, 32, v9
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v8, -1, v8
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v7, 32, v7
-; GFX11TRUE16-NEXT: v_min_u32_e32 v6, v10, v6
+; GFX11TRUE16-NEXT: v_min_u32_e32 v7, v10, v7
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_min_u32_e32 v9, v11, v9
-; GFX11TRUE16-NEXT: v_min_u32_e32 v7, v8, v7
+; GFX11TRUE16-NEXT: v_min_u32_e32 v6, v6, v8
+; GFX11TRUE16-NEXT: v_min_u32_e32 v8, v11, v9
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v6, v[0:1]
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[2:3], v9, v[2:3]
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v7, v[0:1]
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[4:5], v6, v[4:5]
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[4:5], v7, v[4:5]
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[2:3], v8, v[2:3]
; GFX11TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_min_u32_e32 v4, 1, v4
; GFX11TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_min_u32_e32 v1, 1, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v1, v5, v4
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v4, 32, v6
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v3, 32, v6
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v4, 32, v9
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v3, 32, v7
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_or_b32_e32 v1, v5, v1
+; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v5, 32, v8
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v5, 32, v7
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v3
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_ldexp_f32 v2, v2, v4
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v4
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v5
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11TRUE16-NEXT: v_ldexp_f32 v2, v2, v5
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v5, v6, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_sitofp_v3i64_to_v3bf16:
@@ -37547,81 +37353,79 @@ define <4 x bfloat> @v_sitofp_v4i64_to_v4bf16(<4 x i64> %x) {
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v8, 32, v8
; GFX11TRUE16-NEXT: v_cls_i32_e32 v12, v3
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v11, 32, v11
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_min_u32_e32 v8, v9, v8
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v9, -1, v10
; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v10, 31, v13
; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v13, 31, v15
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v12, -1, v12
; GFX11TRUE16-NEXT: v_lshlrev_b64 v[6:7], v8, v[6:7]
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_min_u32_e32 v9, v9, v11
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v10, 32, v10
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v13, 32, v13
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_lshlrev_b64 v[4:5], v9, v[4:5]
; GFX11TRUE16-NEXT: v_min_u32_e32 v6, 1, v6
-; GFX11TRUE16-NEXT: v_min_u32_e32 v10, v12, v10
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_min_u32_e32 v4, 1, v4
; GFX11TRUE16-NEXT: v_or_b32_e32 v6, v7, v6
; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v7, 32, v8
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[2:3], v10, v[2:3]
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_or_b32_e32 v4, v5, v4
; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v5, 32, v9
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v6, v6
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v4, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, v3, v2
; GFX11TRUE16-NEXT: v_ldexp_f32 v4, v4, v5
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v5, 32, v10
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v14, -1, v14
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_ldexp_f32 v2, v2, v5
; GFX11TRUE16-NEXT: v_min_u32_e32 v11, v14, v13
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfe_u32 v9, v2, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v11, v[0:1]
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v3, 32, v11
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v6, v7
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v12, -1, v12
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_min_u32_e32 v10, v12, v10
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[2:3], v10, v[2:3]
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v5, 32, v10
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
+; GFX11TRUE16-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v3, 32, v11
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v3
; GFX11TRUE16-NEXT: v_bfe_u32 v3, v4, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_ldexp_f32 v2, v2, v5
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v9, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v9, v2, 16, 1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v8, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v10, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v9, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v10, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v3, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v5.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_sitofp_v4i64_to_v4bf16:
@@ -37915,13 +37719,11 @@ define <2 x bfloat> @v_uitofp_v2i16_to_v2bf16(<2 x i16> %x) {
; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_v2i16_to_v2bf16:
@@ -38083,30 +37885,28 @@ define <3 x bfloat> @v_uitofp_v3i16_to_v3bf16(<3 x i16> %x) {
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v3
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v5, v6, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_v3i16_to_v3bf16:
@@ -38297,46 +38097,44 @@ define <4 x bfloat> @v_uitofp_v4i16_to_v4bf16(<4 x i16> %x) {
; GFX11TRUE16-LABEL: v_uitofp_v4i16_to_v4bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.h
-; GFX11TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v4, v2
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v3
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v4, v3
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.h
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v3, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v10, v3, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v4
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v2
-; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v10, v2, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v0
+; GFX11TRUE16-NEXT: v_bfe_u32 v0, v2, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v2, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v0, v5, vcc_lo
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11TRUE16-NEXT: v_add3_u32 v0, v10, v3, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v10, v2, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v3
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v11, vcc_lo
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v11, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v5.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_v4i16_to_v4bf16:
@@ -38575,12 +38373,11 @@ define <2 x bfloat> @v_uitofp_v2i32_to_v2bf16(<2 x i32> %x) {
; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_v2i32_to_v2bf16:
@@ -38726,30 +38523,28 @@ define <3 x bfloat> @v_uitofp_v3i32_to_v3bf16(<3 x i32> %x) {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v2
-; GFX11TRUE16-NEXT: v_add3_u32 v5, v6, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v3, vcc_lo
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_v3i32_to_v3bf16:
@@ -38926,36 +38721,32 @@ define <4 x bfloat> @v_uitofp_v4i32_to_v4bf16(<4 x i32> %x) {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v3, v3
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v3, v3
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v4, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX11TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_bfe_u32 v1, v3, 16, 1
; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v10, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v3, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v4, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v9, v10, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v6, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v2, v3
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_v4i32_to_v4bf16:
@@ -39352,12 +39143,11 @@ define <2 x bfloat> @v_uitofp_v2i64_to_v2bf16(<2 x i64> %x) {
; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_v2i64_to_v2bf16:
@@ -39650,8 +39440,8 @@ define <3 x bfloat> @v_uitofp_v3i64_to_v3bf16(<3 x i64> %x) {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v6, v1
-; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v7, v3
-; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v8, v5
+; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v7, v5
+; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v8, v3
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_min_u32_e32 v6, 32, v6
; GFX11TRUE16-NEXT: v_min_u32_e32 v7, 32, v7
@@ -39659,50 +39449,47 @@ define <3 x bfloat> @v_uitofp_v3i64_to_v3bf16(<3 x i64> %x) {
; GFX11TRUE16-NEXT: v_min_u32_e32 v8, 32, v8
; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v6, v[0:1]
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[2:3], v7, v[2:3]
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[4:5], v8, v[4:5]
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[4:5], v7, v[4:5]
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[2:3], v8, v[2:3]
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX11TRUE16-NEXT: v_min_u32_e32 v4, 1, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11TRUE16-NEXT: v_min_u32_e32 v1, 1, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_or_b32_e32 v1, v5, v4
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, v3, v2
; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v3, 32, v6
; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v4, 32, v7
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_or_b32_e32 v1, v5, v1
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v5, 32, v8
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v3
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_ldexp_f32 v2, v2, v4
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v4
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v5
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11TRUE16-NEXT: v_ldexp_f32 v2, v2, v5
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v5, v6, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_v3i64_to_v3bf16:
@@ -40131,17 +39918,13 @@ define <4 x bfloat> @v_uitofp_v4i64_to_v4bf16(<4 x i64> %x) {
; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX11TRUE16-NEXT: v_add3_u32 v5, v9, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v10, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v3, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_v4i64_to_v4bf16:
@@ -46361,13 +46144,12 @@ define <2 x bfloat> @v_fma_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat>
; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v3, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v0, v4, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v5, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fma_v2bf16:
@@ -46590,40 +46372,40 @@ define <3 x bfloat> @v_fma_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat>
; GFX11TRUE16-LABEL: v_fma_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_fmac_f32 v4, v0, v2 :: v_dual_fmac_f32 v5, v1, v3
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v6, v8, v7
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v5, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v0, v6, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v6
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v6, v8, v7 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v4, v0, v2
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v6, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v3, v5, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v5
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v5, v1, v3
+; GFX11TRUE16-NEXT: v_bfe_u32 v1, v6, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v6
+; GFX11TRUE16-NEXT: v_bfe_u32 v0, v5, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v6, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
+; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v5, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v0, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fma_v3bf16:
@@ -46908,47 +46690,47 @@ define <4 x bfloat> @v_fma_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat>
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v5, v1, v3
; GFX11TRUE16-NEXT: v_dual_fmac_f32 v6, v8, v7 :: v_dual_lshlrev_b32 v7, 16, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
-; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v5, 16, 1
+; GFX11TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v9, v6, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
+; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v5, 16, 1
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v3, v0, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v7, v10, v8
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v5, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v1, v0, v2
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v5, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v6
+; GFX11TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
; GFX11TRUE16-NEXT: v_bfe_u32 v0, v7, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v7
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v6
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11TRUE16-NEXT: v_add3_u32 v5, v9, v3, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v7, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v9, v6, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v0, v10, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11TRUE16-NEXT: v_add3_u32 v5, v9, v1, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v3, v2
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fma_v4bf16:
@@ -47298,13 +47080,12 @@ define <2 x bfloat> @v_fmuladd_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfl
; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v3, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v0, v4, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v5, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fmuladd_v2bf16:
@@ -47539,40 +47320,40 @@ define <3 x bfloat> @v_fmuladd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfl
; GFX11TRUE16-LABEL: v_fmuladd_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_fmac_f32 v4, v0, v2 :: v_dual_fmac_f32 v5, v1, v3
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v6, v8, v7
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v5, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v0, v6, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v6
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v6, v8, v7 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v4, v0, v2
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v6, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v3, v5, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v5
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v5, v1, v3
+; GFX11TRUE16-NEXT: v_bfe_u32 v1, v6, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v6
+; GFX11TRUE16-NEXT: v_bfe_u32 v0, v5, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v6, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
+; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v5, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v0, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fmuladd_v3bf16:
@@ -47873,47 +47654,47 @@ define <4 x bfloat> @v_fmuladd_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfl
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v5, v1, v3
; GFX11TRUE16-NEXT: v_dual_fmac_f32 v6, v8, v7 :: v_dual_lshlrev_b32 v7, 16, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
-; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v5, 16, 1
+; GFX11TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v9, v6, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
+; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v5, 16, 1
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v3, v0, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v7, v10, v8
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v5, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v1, v0, v2
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v5, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v6
+; GFX11TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
; GFX11TRUE16-NEXT: v_bfe_u32 v0, v7, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v7
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v6
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11TRUE16-NEXT: v_add3_u32 v5, v9, v3, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v7, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v9, v6, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v0, v10, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11TRUE16-NEXT: v_add3_u32 v5, v9, v1, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v3, v2
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.h
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fmuladd_v4bf16:
diff --git a/llvm/test/CodeGen/AMDGPU/bitcast_vector_bigint.ll b/llvm/test/CodeGen/AMDGPU/bitcast_vector_bigint.ll
index 6b9016df5cd89..f788cd663b309 100644
--- a/llvm/test/CodeGen/AMDGPU/bitcast_vector_bigint.ll
+++ b/llvm/test/CodeGen/AMDGPU/bitcast_vector_bigint.ll
@@ -201,11 +201,6 @@ define <10 x i16> @bitcast_i160_to_v10i16(i160 %int) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v5, v0
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v6, v2
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: bitcast_i160_to_v10i16:
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index 0ceb9019eb990..c3b14e8829042 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -9127,11 +9127,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v0, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -9557,13 +9555,11 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.h
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -10034,6 +10030,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
+; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB28_3: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-TRUE16-NEXT: ; Child Loop BB28_4 Depth 2
@@ -10057,9 +10055,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v11, v13, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v4, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v5
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-TRUE16-NEXT: .LBB28_4: ; Parent Loop BB28_3 Depth=1
@@ -10090,6 +10086,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_3
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v4
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -10734,11 +10731,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v0, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -11164,13 +11159,11 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.h
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -11597,11 +11590,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v0, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -12027,13 +12018,11 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.h
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -12453,13 +12442,11 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.h
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index cad4c39eaf39f..f7a1fb35c8106 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -7426,12 +7426,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v0, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -7581,11 +7579,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v0, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -7955,13 +7951,12 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v5
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -8101,13 +8096,11 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.h
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -8495,9 +8488,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v11, v13, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v4, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v5
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-TRUE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
@@ -8728,6 +8720,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
+; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-TRUE16-NEXT: ; Child Loop BB21_4 Depth 2
@@ -8751,9 +8745,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v11, v13, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v4, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v5
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-TRUE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
@@ -8784,6 +8776,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_3
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v4
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index 6275afd2c6994..8ac6353133e72 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -7426,12 +7426,10 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v0, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -7581,11 +7579,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v0, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -7955,13 +7951,12 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v5
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -8101,13 +8096,11 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.h
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -8495,9 +8488,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v11, v13, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v4, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v5
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX12-TRUE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
@@ -8728,6 +8720,8 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
+; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB21_3: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-TRUE16-NEXT: ; Child Loop BB21_4 Depth 2
@@ -8751,9 +8745,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v11, v13, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v4, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v5
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v6
; GFX11-TRUE16-NEXT: .LBB21_4: ; Parent Loop BB21_3 Depth=1
@@ -8784,6 +8776,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_3
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v4
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll b/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll
index d1a1112777aae..861621bd92af1 100644
--- a/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll
@@ -476,25 +476,14 @@ define void @undef_lo2_v4f16(<2 x half> %arg0) {
; GFX11-FAKE16-NEXT: ;;#ASMEND
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-SDAG-LABEL: undef_lo2_v4f16:
-; GFX11-TRUE16-SDAG: ; %bb.0:
-; GFX11-TRUE16-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-SDAG-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-SDAG-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
-; GFX11-TRUE16-SDAG-NEXT: ;;#ASMSTART
-; GFX11-TRUE16-SDAG-NEXT: ; use v[0:1]
-; GFX11-TRUE16-SDAG-NEXT: ;;#ASMEND
-; GFX11-TRUE16-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-GISEL-LABEL: undef_lo2_v4f16:
-; GFX11-TRUE16-GISEL: ; %bb.0:
-; GFX11-TRUE16-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-GISEL-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-GISEL-NEXT: ;;#ASMSTART
-; GFX11-TRUE16-GISEL-NEXT: ; use v[0:1]
-; GFX11-TRUE16-GISEL-NEXT: ;;#ASMEND
-; GFX11-TRUE16-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: undef_lo2_v4f16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: ;;#ASMSTART
+; GFX11-TRUE16-NEXT: ; use v[0:1]
+; GFX11-TRUE16-NEXT: ;;#ASMEND
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
%undef.lo = shufflevector <2 x half> %arg0, <2 x half> poison, <4 x i32> <i32 1, i32 1, i32 2, i32 3>
call void asm sideeffect "; use $0", "v"(<4 x half> %undef.lo);
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/build_vector.gfx11plus.ll b/llvm/test/CodeGen/AMDGPU/build_vector.gfx11plus.ll
new file mode 100644
index 0000000000000..7b5869d9be7e4
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/build_vector.gfx11plus.ll
@@ -0,0 +1,145 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -stop-after=amdgpu-isel | FileCheck %s --check-prefixes=GFX11,GFX11-REAL16
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 -stop-after=amdgpu-isel | FileCheck %s --check-prefixes=GFX11,GFX11-FAKE16
+
+define amdgpu_kernel void @build_vector_bfi (ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %out) {
+ ; GFX11-REAL16-LABEL: name: build_vector_bfi
+ ; GFX11-REAL16: bb.0.entry:
+ ; GFX11-REAL16-NEXT: liveins: $vgpr0, $sgpr4_sgpr5
+ ; GFX11-REAL16-NEXT: {{ $}}
+ ; GFX11-REAL16-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+ ; GFX11-REAL16-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+ ; GFX11-REAL16-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s128) from %ir.a.kernarg.offset, align 4, addrspace 4)
+ ; GFX11-REAL16-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 52, 0 :: (dereferenceable invariant load (s64) from %ir.a.kernarg.offset + 16, align 4, addrspace 4)
+ ; GFX11-REAL16-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
+ ; GFX11-REAL16-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM]].sub0
+ ; GFX11-REAL16-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub3
+ ; GFX11-REAL16-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub2
+ ; GFX11-REAL16-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub1
+ ; GFX11-REAL16-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub0
+ ; GFX11-REAL16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE killed [[COPY7]], %subreg.sub0, killed [[COPY6]], %subreg.sub1
+ ; GFX11-REAL16-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE killed [[COPY5]], %subreg.sub0, killed [[COPY4]], %subreg.sub1
+ ; GFX11-REAL16-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE killed [[COPY3]], %subreg.sub0, killed [[COPY2]], %subreg.sub1
+ ; GFX11-REAL16-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX11-REAL16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1023
+ ; GFX11-REAL16-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY1]](s32), killed [[S_MOV_B32_]], implicit $exec
+ ; GFX11-REAL16-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 2
+ ; GFX11-REAL16-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = nuw nsw V_LSHLREV_B32_e64 killed [[S_MOV_B32_1]], killed [[V_AND_B32_e64_]], implicit $exec
+ ; GFX11-REAL16-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR killed [[REG_SEQUENCE]], [[V_LSHLREV_B32_e64_]], 0, 0, implicit $exec :: (load (s32) from %ir.in.gep1, addrspace 1)
+ ; GFX11-REAL16-NEXT: [[GLOBAL_LOAD_DWORD_SADDR1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR killed [[REG_SEQUENCE1]], [[V_LSHLREV_B32_e64_]], 0, 0, implicit $exec :: (load (s32) from %ir.in.gep2, addrspace 1)
+ ; GFX11-REAL16-NEXT: [[COPY8:%[0-9]+]]:vgpr_16 = COPY [[GLOBAL_LOAD_DWORD_SADDR]].lo16
+ ; GFX11-REAL16-NEXT: [[COPY9:%[0-9]+]]:vgpr_16 = COPY [[GLOBAL_LOAD_DWORD_SADDR1]].hi16
+ ; GFX11-REAL16-NEXT: [[V_MOV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_MOV_B16_t16_e64 0, 0, 0, implicit $exec
+ ; GFX11-REAL16-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vgpr_32 = REG_SEQUENCE killed [[COPY9]], %subreg.lo16, killed [[V_MOV_B16_t16_e64_]], %subreg.hi16
+ ; GFX11-REAL16-NEXT: [[COPY10:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE3]].lo16
+ ; GFX11-REAL16-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:vgpr_32 = REG_SEQUENCE killed [[COPY8]], %subreg.lo16, killed [[COPY10]], %subreg.hi16
+ ; GFX11-REAL16-NEXT: GLOBAL_STORE_DWORD_SADDR killed [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE4]], killed [[REG_SEQUENCE2]], 0, 0, implicit $exec :: (store (s32) into %ir.3, addrspace 1)
+ ; GFX11-REAL16-NEXT: S_ENDPGM 0
+ ;
+ ; GFX11-FAKE16-LABEL: name: build_vector_bfi
+ ; GFX11-FAKE16: bb.0.entry:
+ ; GFX11-FAKE16-NEXT: liveins: $vgpr0, $sgpr4_sgpr5
+ ; GFX11-FAKE16-NEXT: {{ $}}
+ ; GFX11-FAKE16-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+ ; GFX11-FAKE16-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+ ; GFX11-FAKE16-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s128) from %ir.a.kernarg.offset, align 4, addrspace 4)
+ ; GFX11-FAKE16-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 52, 0 :: (dereferenceable invariant load (s64) from %ir.a.kernarg.offset + 16, align 4, addrspace 4)
+ ; GFX11-FAKE16-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
+ ; GFX11-FAKE16-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM]].sub0
+ ; GFX11-FAKE16-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub3
+ ; GFX11-FAKE16-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub2
+ ; GFX11-FAKE16-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub1
+ ; GFX11-FAKE16-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub0
+ ; GFX11-FAKE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE killed [[COPY7]], %subreg.sub0, killed [[COPY6]], %subreg.sub1
+ ; GFX11-FAKE16-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE killed [[COPY5]], %subreg.sub0, killed [[COPY4]], %subreg.sub1
+ ; GFX11-FAKE16-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE killed [[COPY3]], %subreg.sub0, killed [[COPY2]], %subreg.sub1
+ ; GFX11-FAKE16-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX11-FAKE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1023
+ ; GFX11-FAKE16-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY1]](s32), killed [[S_MOV_B32_]], implicit $exec
+ ; GFX11-FAKE16-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 2
+ ; GFX11-FAKE16-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = nuw nsw V_LSHLREV_B32_e64 killed [[S_MOV_B32_1]], killed [[V_AND_B32_e64_]], implicit $exec
+ ; GFX11-FAKE16-NEXT: [[GLOBAL_LOAD_DWORD_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR killed [[REG_SEQUENCE]], [[V_LSHLREV_B32_e64_]], 0, 0, implicit $exec :: (load (s32) from %ir.in.gep1, addrspace 1)
+ ; GFX11-FAKE16-NEXT: [[GLOBAL_LOAD_DWORD_SADDR1:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR killed [[REG_SEQUENCE1]], [[V_LSHLREV_B32_e64_]], 0, 0, implicit $exec :: (load (s32) from %ir.in.gep2, addrspace 1)
+ ; GFX11-FAKE16-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY killed [[GLOBAL_LOAD_DWORD_SADDR]]
+ ; GFX11-FAKE16-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; GFX11-FAKE16-NEXT: [[V_BFI_B32_e64_:%[0-9]+]]:vgpr_32 = V_BFI_B32_e64 killed [[S_MOV_B32_2]], killed [[COPY8]], killed [[GLOBAL_LOAD_DWORD_SADDR1]], implicit $exec
+ ; GFX11-FAKE16-NEXT: GLOBAL_STORE_DWORD_SADDR killed [[V_MOV_B32_e32_]], killed [[V_BFI_B32_e64_]], killed [[REG_SEQUENCE2]], 0, 0, implicit $exec :: (store (s32) into %ir.3, addrspace 1)
+ ; GFX11-FAKE16-NEXT: S_ENDPGM 0
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %in.gep1 = getelementptr i32, ptr addrspace(1) %a, i32 %tid
+ %in.gep2 = getelementptr i32, ptr addrspace(1) %b, i32 %tid
+ %load.i32.a = load i32, ptr addrspace(1) %in.gep1
+ %load.i32.b = load i32, ptr addrspace(1) %in.gep2
+ %load.i16.a = trunc i32 %load.i32.a to i16
+ %load.i32.b.1 = lshr i32 %load.i32.b, 16
+ %load.i16.b = trunc i32 %load.i32.b.1 to i16
+ %ins.0 = insertelement <2 x i16> poison, i16 %load.i16.a, i32 0
+ %ins.1 = insertelement <2 x i16> %ins.0, i16 %load.i16.b, i32 1
+ store <2 x i16> %ins.1, ptr addrspace(1) %out
+ ret void
+}
+
+define amdgpu_kernel void @build_vector_and (ptr addrspace(1) %a, ptr addrspace(1) %out) {
+ ; GFX11-REAL16-LABEL: name: build_vector_and
+ ; GFX11-REAL16: bb.0.entry:
+ ; GFX11-REAL16-NEXT: liveins: $vgpr0, $sgpr4_sgpr5
+ ; GFX11-REAL16-NEXT: {{ $}}
+ ; GFX11-REAL16-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+ ; GFX11-REAL16-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+ ; GFX11-REAL16-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s128) from %ir.a.kernarg.offset, align 4, addrspace 4)
+ ; GFX11-REAL16-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub1
+ ; GFX11-REAL16-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub0
+ ; GFX11-REAL16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE killed [[COPY3]], %subreg.sub0, killed [[COPY2]], %subreg.sub1
+ ; GFX11-REAL16-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub3
+ ; GFX11-REAL16-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub2
+ ; GFX11-REAL16-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE killed [[COPY5]], %subreg.sub0, killed [[COPY4]], %subreg.sub1
+ ; GFX11-REAL16-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX11-REAL16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1023
+ ; GFX11-REAL16-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY1]](s32), killed [[S_MOV_B32_]], implicit $exec
+ ; GFX11-REAL16-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+ ; GFX11-REAL16-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = nuw nsw V_LSHLREV_B32_e64 killed [[S_MOV_B32_1]], killed [[V_AND_B32_e64_]], implicit $exec
+ ; GFX11-REAL16-NEXT: [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 killed [[REG_SEQUENCE]], killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $exec :: (load (s16) from %ir.in.gep1, addrspace 1)
+ ; GFX11-REAL16-NEXT: [[V_MOV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_MOV_B16_t16_e64 0, 1, 0, implicit $exec
+ ; GFX11-REAL16-NEXT: [[V_ADD_NC_U16_t16_e64_:%[0-9]+]]:vgpr_16 = V_ADD_NC_U16_t16_e64 0, killed [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], 0, killed [[V_MOV_B16_t16_e64_]], 0, 0, implicit $exec
+ ; GFX11-REAL16-NEXT: [[V_MOV_B16_t16_e64_1:%[0-9]+]]:vgpr_16 = V_MOV_B16_t16_e64 0, 0, 0, implicit $exec
+ ; GFX11-REAL16-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vgpr_32 = REG_SEQUENCE killed [[V_ADD_NC_U16_t16_e64_]], %subreg.lo16, killed [[V_MOV_B16_t16_e64_1]], %subreg.hi16
+ ; GFX11-REAL16-NEXT: GLOBAL_STORE_DWORD_SADDR killed [[V_MOV_B32_e32_]], killed [[REG_SEQUENCE2]], killed [[REG_SEQUENCE1]], 0, 0, implicit $exec :: (store (s32) into %ir.2, addrspace 1)
+ ; GFX11-REAL16-NEXT: S_ENDPGM 0
+ ;
+ ; GFX11-FAKE16-LABEL: name: build_vector_and
+ ; GFX11-FAKE16: bb.0.entry:
+ ; GFX11-FAKE16-NEXT: liveins: $vgpr0, $sgpr4_sgpr5
+ ; GFX11-FAKE16-NEXT: {{ $}}
+ ; GFX11-FAKE16-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+ ; GFX11-FAKE16-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+ ; GFX11-FAKE16-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s128) from %ir.a.kernarg.offset, align 4, addrspace 4)
+ ; GFX11-FAKE16-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub1
+ ; GFX11-FAKE16-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub0
+ ; GFX11-FAKE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE killed [[COPY3]], %subreg.sub0, killed [[COPY2]], %subreg.sub1
+ ; GFX11-FAKE16-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub3
+ ; GFX11-FAKE16-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub2
+ ; GFX11-FAKE16-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE killed [[COPY5]], %subreg.sub0, killed [[COPY4]], %subreg.sub1
+ ; GFX11-FAKE16-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GFX11-FAKE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1023
+ ; GFX11-FAKE16-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY1]](s32), killed [[S_MOV_B32_]], implicit $exec
+ ; GFX11-FAKE16-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+ ; GFX11-FAKE16-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = nuw nsw V_LSHLREV_B32_e64 killed [[S_MOV_B32_1]], killed [[V_AND_B32_e64_]], implicit $exec
+ ; GFX11-FAKE16-NEXT: [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR killed [[REG_SEQUENCE]], killed [[V_LSHLREV_B32_e64_]], 0, 0, implicit $exec :: (load (s16) from %ir.in.gep1, addrspace 1)
+ ; GFX11-FAKE16-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+ ; GFX11-FAKE16-NEXT: [[V_ADD_NC_U16_fake16_e64_:%[0-9]+]]:vgpr_32 = V_ADD_NC_U16_fake16_e64 0, killed [[GLOBAL_LOAD_USHORT_SADDR]], 0, killed [[S_MOV_B32_2]], 0, 0, implicit $exec
+ ; GFX11-FAKE16-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 65535, implicit $exec
+ ; GFX11-FAKE16-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 killed [[V_MOV_B32_e32_1]], killed [[V_ADD_NC_U16_fake16_e64_]], implicit $exec
+ ; GFX11-FAKE16-NEXT: GLOBAL_STORE_DWORD_SADDR killed [[V_MOV_B32_e32_]], killed [[V_AND_B32_e64_1]], killed [[REG_SEQUENCE1]], 0, 0, implicit $exec :: (store (s32) into %ir.2, addrspace 1)
+ ; GFX11-FAKE16-NEXT: S_ENDPGM 0
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ %in.gep1 = getelementptr i16, ptr addrspace(1) %a, i32 %tid
+ %load.i16.a = load i16, ptr addrspace(1) %in.gep1
+ %add = add i16 %load.i16.a, 1
+ %ins.0 = insertelement <2 x i16> poison, i16 %add, i32 0
+ %ins.1 = insertelement <2 x i16> %ins.0, i16 0, i32 1
+ store <2 x i16> %ins.1, ptr addrspace(1) %out
+ ret void
+}
+
diff --git a/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll b/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
index d4581672dab39..d374ed072cdc6 100644
--- a/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
+++ b/llvm/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
@@ -834,8 +834,9 @@ define <2 x i16> @chain_hi_to_lo_group_other_dep_multi_chain(ptr addrspace(3) %p
; GFX11-TRUE16-NEXT: ds_load_u16_d16_hi v0, v0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_pk_add_u16 v0, v0, 12 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.h
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: chain_hi_to_lo_group_other_dep_multi_chain:
@@ -967,8 +968,9 @@ define <2 x i16> @chain_hi_to_lo_global_other_dep(ptr addrspace(1) %ptr) {
; GFX11-TRUE16-NEXT: global_load_d16_hi_b16 v0, v[0:1], off glc dlc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_pk_add_u16 v0, v0, 12 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.h
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: chain_hi_to_lo_global_other_dep:
@@ -1038,11 +1040,11 @@ define <2 x i16> @chain_hi_to_lo_flat_other_dep(ptr addrspace(0) %ptr) {
; GFX11-TRUE16-NEXT: flat_load_d16_b16 v2, v[0:1] offset:2 glc dlc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: flat_load_d16_hi_b16 v0, v[0:1] glc dlc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_pk_add_u16 v0, v0, 12 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.h
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: chain_hi_to_lo_flat_other_dep:
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
index f26b72027a784..0c7dc74d95e49 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
@@ -3116,23 +3116,21 @@ define <2 x bfloat> @fmul_select_v2bf16_test3(<2 x bfloat> %x, <2 x i32> %bool.a
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v2, v4, v3
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v3
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: fmul_select_v2bf16_test3:
@@ -3181,23 +3179,21 @@ define <2 x bfloat> @fmul_select_v2bf16_test3(<2 x bfloat> %x, <2 x i32> %bool.a
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v2, v4, v3
; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v3
-; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-FAKE16-LABEL: fmul_select_v2bf16_test3:
@@ -3325,23 +3321,21 @@ define <2 x bfloat> @fmul_select_v2bf16_test4(<2 x bfloat> %x, <2 x i32> %bool.a
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v2, v4, v3
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v3
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: fmul_select_v2bf16_test4:
@@ -3390,23 +3384,21 @@ define <2 x bfloat> @fmul_select_v2bf16_test4(<2 x bfloat> %x, <2 x i32> %bool.a
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v2, v4, v3
; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v3
-; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX11-GISEL-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-GISEL-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
-; GFX11-GISEL-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX11-GISEL-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-FAKE16-LABEL: fmul_select_v2bf16_test4:
diff --git a/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll b/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll
index 85e56a243cdc9..9c59b4236cae4 100644
--- a/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll
+++ b/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll
@@ -462,11 +462,19 @@ define i32 @divergent_vec_i16_LH(i16 %a, i32 %b) {
; GFX906-NEXT: v_bfi_b32 v0, s4, v0, v1
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: divergent_vec_i16_LH:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: divergent_vec_i16_LH:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v1
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: divergent_vec_i16_LH:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%shift = lshr i32 %b, 16
%tr = trunc i32 %shift to i16
%tmp = insertelement <2 x i16> poison, i16 %a, i32 0
diff --git a/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll b/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll
index d8f81db70e309..5d184b17f32e6 100644
--- a/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll
@@ -630,29 +630,28 @@ define amdgpu_kernel void @v_fabs_fold_self_v2bf16(ptr addrspace(1) %out, ptr ad
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_mul_f32 v2, v1, v2 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_mul_f32 v2, v1, v2 :: v_dual_and_b32 v3, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX11-TRUE16-NEXT: global_store_b32 v2, v0, s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
;
@@ -813,31 +812,30 @@ define amdgpu_kernel void @v_fabs_fold_v2bf16(ptr addrspace(1) %out, ptr addrspa
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[2:3]
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s4, 0xffff0000
+; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s4, 16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.h
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_mul_f32_e32 v2, s2, v1
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.l
-; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s4, 16
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0x7fff, v0.h
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s4, 0xffff0000
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mul_f32_e32 v0, s2, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX11-TRUE16-NEXT: global_store_b32 v2, v0, s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll b/llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll
index 450d66767600b..1a3c8febea865 100644
--- a/llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcopysign.bf16.ll
@@ -4045,10 +4045,8 @@ define <2 x bfloat> @v_copysign_out_v2bf16_mag_v2f32_sign_v2bf16(<2 x float> %ma
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fff7fff, v0, v2
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fff7fff, v1, v2
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_copysign_out_v2bf16_mag_v2f32_sign_v2bf16:
@@ -4274,10 +4272,8 @@ define <2 x bfloat> @v_copysign_out_v2bf16_mag_v2f64_sign_v2bf16(<2 x double> %m
; GFX11TRUE16-NEXT: v_cmp_u_f64_e32 vcc_lo, v[2:3], v[2:3]
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v6, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fff7fff, v0, v4
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fff7fff, v1, v4
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_copysign_out_v2bf16_mag_v2f64_sign_v2bf16:
@@ -4439,10 +4435,8 @@ define <2 x bfloat> @v_copysign_out_v2bf16_mag_v2bf16_sign_v2f32(<2 x bfloat> %m
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fff7fff, v0, v1
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.h
+; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fff7fff, v0, v2
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_copysign_out_v2bf16_mag_v2bf16_sign_v2f32:
@@ -5842,29 +5836,26 @@ define <3 x bfloat> @v_copysign_out_v3bf16_mag_v3f32_sign_v3bf16(<3 x float> %ma
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11TRUE16-NEXT: v_add3_u32 v7, v8, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v9, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fff7fff, v0, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0x7fff7fff, v1, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v10, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fff7fff, v1, v3
+; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0x7fff7fff, v2, v4
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_copysign_out_v3bf16_mag_v3f32_sign_v3bf16:
@@ -6124,65 +6115,65 @@ define <3 x bfloat> @v_copysign_out_v3bf16_mag_v3f64_sign_v3bf16(<3 x double> %m
; GFX11TRUE16-LABEL: v_copysign_out_v3bf16_mag_v3f64_sign_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_cvt_f32_f64_e32 v16, v[4:5]
; GFX11TRUE16-NEXT: v_cvt_f32_f64_e32 v14, v[0:1]
-; GFX11TRUE16-NEXT: v_cvt_f32_f64_e32 v15, v[2:3]
+; GFX11TRUE16-NEXT: v_cvt_f32_f64_e32 v15, v[4:5]
+; GFX11TRUE16-NEXT: v_cvt_f32_f64_e32 v16, v[2:3]
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cvt_f64_f32_e32 v[12:13], v16
; GFX11TRUE16-NEXT: v_cvt_f64_f32_e32 v[8:9], v14
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_cvt_f64_f32_e32 v[10:11], v15
-; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 1, v16
-; GFX11TRUE16-NEXT: v_cmp_gt_f64_e64 s4, |v[4:5]|, |v[12:13]|
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cmp_gt_f64_e64 s2, |v[0:1]|, |v[8:9]|
-; GFX11TRUE16-NEXT: v_cmp_gt_f64_e64 s3, |v[2:3]|, |v[10:11]|
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cvt_f64_f32_e32 v[12:13], v16
+; GFX11TRUE16-NEXT: v_cmp_gt_f64_e64 s3, |v[0:1]|, |v[8:9]|
; GFX11TRUE16-NEXT: v_cmp_nlg_f64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX11TRUE16-NEXT: v_cmp_nlg_f64_e64 s1, v[4:5], v[12:13]
-; GFX11TRUE16-NEXT: v_cmp_nlg_f64_e64 s0, v[2:3], v[10:11]
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v10, -1, 1, s4
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v8, -1, 1, s2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cmp_nlg_f64_e64 s0, v[4:5], v[10:11]
+; GFX11TRUE16-NEXT: v_cmp_nlg_f64_e64 s1, v[2:3], v[12:13]
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v8, -1, 1, s3
+; GFX11TRUE16-NEXT: v_cmp_gt_f64_e64 s3, |v[4:5]|, |v[10:11]|
+; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v8, v14, v8
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v9, -1, 1, s3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cmp_gt_f64_e64 s3, |v[2:3]|, |v[12:13]|
+; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v9, v15, v9
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v10, -1, 1, s3
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v10, v16, v10
; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 1, v14
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v8, v14, v8
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v9, v15, v9
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_cmp_eq_u32_e64 s3, 1, v19
; GFX11TRUE16-NEXT: s_or_b32 vcc_lo, vcc_lo, s3
; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v8, v8, v14 :: v_dual_and_b32 v17, 1, v15
-; GFX11TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v18
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_cmp_eq_u32_e64 s2, 1, v17
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
-; GFX11TRUE16-NEXT: s_or_b32 vcc_lo, s1, vcc_lo
-; GFX11TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v8
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v10, v16, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[0:1]
+; GFX11TRUE16-NEXT: s_or_b32 vcc_lo, s0, s2
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v9, v9, v15 :: v_dual_and_b32 v18, 1, v16
+; GFX11TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v8
; GFX11TRUE16-NEXT: v_add3_u32 v8, v11, v8, 0x7fff
-; GFX11TRUE16-NEXT: s_or_b32 s0, s0, s2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v9, v9, v15, s0
-; GFX11TRUE16-NEXT: v_bfe_u32 v11, v10, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v10
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cmp_eq_u32_e64 s4, 1, v18
; GFX11TRUE16-NEXT: v_bfe_u32 v12, v9, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v9
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v9
+; GFX11TRUE16-NEXT: s_or_b32 vcc_lo, s1, s4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_add3_u32 v9, v12, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v13, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f64_e32 vcc_lo, v[2:3], v[2:3]
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v11, v10, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v9, v14, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v10, v16, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[0:1]
+; GFX11TRUE16-NEXT: v_bfe_u32 v14, v10, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v10
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v14, v10, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v15, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f64_e32 vcc_lo, v[4:5], v[4:5]
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fff7fff, v0, v6
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v13, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f64_e32 vcc_lo, v[2:3], v[2:3]
; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0x7fff7fff, v1, v7
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v10, v11, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fff7fff, v2, v6
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_copysign_out_v3bf16_mag_v3f64_sign_v3bf16:
@@ -6387,29 +6378,26 @@ define <3 x bfloat> @v_copysign_out_v3bf16_mag_v3bf16_sign_v3f32(<3 x bfloat> %m
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v4
-; GFX11TRUE16-NEXT: v_add3_u32 v7, v8, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v9, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.h
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fff7fff, v0, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0x7fff7fff, v1, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v10, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0x7fff7fff, v1, v2
+; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fff7fff, v0, v3
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_copysign_out_v3bf16_mag_v3bf16_sign_v3f32:
@@ -7083,15 +7071,12 @@ define <4 x bfloat> @v_copysign_out_v4bf16_mag_v4f32_sign_v4bf16(<4 x float> %ma
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v10, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v2, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fff7fff, v0, v4
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0x7fff7fff, v1, v5
+; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fff7fff, v1, v4
+; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0x7fff7fff, v3, v5
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_copysign_out_v4bf16_mag_v4f32_sign_v4bf16:
@@ -7485,18 +7470,14 @@ define <4 x bfloat> @v_copysign_out_v4bf16_mag_v4f64_sign_v4bf16(<4 x double> %m
; GFX11TRUE16-NEXT: v_cmp_u_f64_e32 vcc_lo, v[0:1], v[0:1]
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v13, v21, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f64_e32 vcc_lo, v[6:7], v[6:7]
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v10, v15, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f64_e32 vcc_lo, v[2:3], v[2:3]
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.h
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v3, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0x7fff7fff, v1, v9
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v12, v20, vcc_lo
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fff7fff, v0, v8
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fff7fff, v2, v8
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_copysign_out_v4bf16_mag_v4f64_sign_v4bf16:
@@ -7767,15 +7748,12 @@ define <4 x bfloat> @v_copysign_out_v4bf16_mag_v4bf16_sign_v4f32(<4 x bfloat> %m
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v10, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v3
-; GFX11TRUE16-NEXT: v_bfi_b32 v3, 0xffff, v4, v5
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fff7fff, v0, v2
-; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0x7fff7fff, v1, v3
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_bfi_b32 v1, 0x7fff7fff, v1, v5
+; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fff7fff, v0, v3
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_copysign_out_v4bf16_mag_v4bf16_sign_v4f32:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
index ae5da3ad094c7..ee87c65c00def 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
@@ -18660,7 +18660,6 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB68_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -18683,10 +18682,9 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -18698,7 +18696,6 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB68_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -18986,7 +18983,6 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB69_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -19009,10 +19005,9 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -19024,7 +19019,6 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB69_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -19321,7 +19315,6 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v0, v[3:4]
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB70_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -19343,10 +19336,9 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v0, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -19358,7 +19350,6 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB70_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -19658,7 +19649,6 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB71_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -19676,13 +19666,12 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -19695,7 +19684,6 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB71_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -19974,7 +19962,6 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB72_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -19992,13 +19979,12 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -20011,7 +19997,6 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB72_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -20303,7 +20288,6 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB73_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -20321,13 +20305,12 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -20340,7 +20323,6 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB73_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -20640,7 +20622,6 @@ define <2 x bfloat> @flat_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB74_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -20663,10 +20644,9 @@ define <2 x bfloat> @flat_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -20678,7 +20658,6 @@ define <2 x bfloat> @flat_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB74_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -20972,7 +20951,6 @@ define void @flat_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB75_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -20990,13 +20968,12 @@ define void @flat_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -21009,7 +20986,6 @@ define void @flat_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB75_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -21297,7 +21273,6 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memory(
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB76_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -21320,10 +21295,9 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memory(
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -21335,7 +21309,6 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memory(
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB76_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -21623,7 +21596,6 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory(ptr %p
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB77_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -21641,13 +21613,12 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory(ptr %p
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -21660,7 +21631,6 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory(ptr %p
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB77_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -21939,7 +21909,6 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB78_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -21962,10 +21931,9 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -21977,7 +21945,6 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB78_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -22265,7 +22232,6 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory_
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB79_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -22283,13 +22249,12 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory_
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -22302,7 +22267,6 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory_
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB79_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
index 6218a5c82afcd..b8a2476dc19b4 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
@@ -16192,11 +16192,10 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -16314,7 +16313,6 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -16337,10 +16335,9 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -16352,7 +16349,6 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -16643,11 +16639,10 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -16765,7 +16760,6 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -16788,10 +16782,9 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -16803,7 +16796,6 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB55_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -17097,11 +17089,10 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -17229,7 +17220,6 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v0, v[3:4]
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -17251,10 +17241,9 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v0, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -17266,7 +17255,6 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB56_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -17564,14 +17552,12 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -17687,7 +17673,6 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -17705,13 +17690,12 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -17724,7 +17708,6 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB57_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -18001,14 +17984,12 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -18124,7 +18105,6 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -18142,13 +18122,12 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -18161,7 +18140,6 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB58_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -18445,14 +18423,12 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -18577,7 +18553,6 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -18595,13 +18570,12 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -18614,7 +18588,6 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB59_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -18916,11 +18889,10 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -19040,7 +19012,6 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -19063,10 +19034,9 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -19078,7 +19048,6 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB60_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -19369,14 +19338,12 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -19494,7 +19461,6 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -19512,13 +19478,12 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -19531,7 +19496,6 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB61_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
index 6eafbb50e4bb9..9830e48a86f06 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
@@ -16192,11 +16192,10 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -16314,7 +16313,6 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -16337,10 +16335,9 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -16352,7 +16349,6 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -16643,11 +16639,10 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -16765,7 +16760,6 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -16788,10 +16782,9 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -16803,7 +16796,6 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB55_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -17097,11 +17089,10 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -17229,7 +17220,6 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v0, v[3:4]
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -17251,10 +17241,9 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v0, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -17266,7 +17255,6 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB56_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -17564,14 +17552,12 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -17687,7 +17673,6 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -17705,13 +17690,12 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -17724,7 +17708,6 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB57_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -18001,14 +17984,12 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -18124,7 +18105,6 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -18142,13 +18122,12 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -18161,7 +18140,6 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB58_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -18445,14 +18423,12 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -18577,7 +18553,6 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -18595,13 +18570,12 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -18614,7 +18588,6 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB59_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -18916,11 +18889,10 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -19040,7 +19012,6 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -19063,10 +19034,9 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -19078,7 +19048,6 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB60_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -19369,14 +19338,12 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -19494,7 +19461,6 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -19512,13 +19478,12 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -19531,7 +19496,6 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB61_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
index 25f29c8c87c96..2c1970220c374 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
@@ -15611,11 +15611,10 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -15733,7 +15732,6 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -15756,10 +15754,9 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -15771,7 +15768,6 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB50_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -16062,11 +16058,10 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -16184,7 +16179,6 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -16207,10 +16201,9 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -16222,7 +16215,6 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB51_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -16516,11 +16508,10 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -16648,7 +16639,6 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v0, v[3:4]
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -16670,10 +16660,9 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v0, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -16685,7 +16674,6 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB52_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -16983,14 +16971,12 @@ define void @flat_agent_atomic_fsub_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -17106,7 +17092,6 @@ define void @flat_agent_atomic_fsub_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -17124,13 +17109,12 @@ define void @flat_agent_atomic_fsub_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -17143,7 +17127,6 @@ define void @flat_agent_atomic_fsub_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB53_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -17420,14 +17403,12 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -17543,7 +17524,6 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -17561,13 +17541,12 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -17580,7 +17559,6 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -17864,14 +17842,12 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -17996,7 +17972,6 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -18014,13 +17989,12 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -18033,7 +18007,6 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB55_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -18335,11 +18308,10 @@ define <2 x bfloat> @flat_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -18459,7 +18431,6 @@ define <2 x bfloat> @flat_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -18482,10 +18453,9 @@ define <2 x bfloat> @flat_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -18497,7 +18467,6 @@ define <2 x bfloat> @flat_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB56_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -18788,14 +18757,12 @@ define void @flat_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -18913,7 +18880,6 @@ define void @flat_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -18931,13 +18897,12 @@ define void @flat_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -18950,7 +18915,6 @@ define void @flat_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB57_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
index 9c1f9d21b9da3..2cad8eeea33cf 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
@@ -20405,7 +20405,6 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB78_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -20428,10 +20427,9 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -20443,7 +20441,6 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB78_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -20784,7 +20781,6 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB79_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -20807,10 +20803,9 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -20822,7 +20817,6 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB79_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -21165,7 +21159,6 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB80_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -21188,10 +21181,9 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -21203,7 +21195,6 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB80_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -21550,7 +21541,6 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB81_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -21568,13 +21558,12 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -21587,7 +21576,6 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB81_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -21917,7 +21905,6 @@ define void @global_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB82_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -21935,13 +21922,12 @@ define void @global_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -21954,7 +21940,6 @@ define void @global_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB82_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -22287,7 +22272,6 @@ define void @global_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB83_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -22305,13 +22289,12 @@ define void @global_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -22324,7 +22307,6 @@ define void @global_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB83_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -22666,7 +22648,6 @@ define <2 x bfloat> @global_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB84_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -22689,10 +22670,9 @@ define <2 x bfloat> @global_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -22704,7 +22684,6 @@ define <2 x bfloat> @global_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB84_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -23050,7 +23029,6 @@ define void @global_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB85_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -23068,13 +23046,12 @@ define void @global_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -23087,7 +23064,6 @@ define void @global_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB85_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -23422,7 +23398,6 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memor
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB86_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -23445,10 +23420,9 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memor
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -23460,7 +23434,6 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memor
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB86_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -23801,7 +23774,6 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory(ptr
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB87_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -23819,13 +23791,12 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory(ptr
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -23838,7 +23809,6 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory(ptr
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB87_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -24168,7 +24138,6 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB88_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -24191,10 +24160,9 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -24206,7 +24174,6 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB88_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -24547,7 +24514,6 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB89_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -24565,13 +24531,12 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -24584,7 +24549,6 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB89_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -24914,7 +24878,6 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__maybe_remote(ptr addrs
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB90_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -24937,10 +24900,9 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__maybe_remote(ptr addrs
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -24952,7 +24914,6 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__maybe_remote(ptr addrs
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB90_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -25293,7 +25254,6 @@ define void @global_agent_atomic_fadd_noret_v2bf16__maybe_remote(ptr addrspace(1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB91_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -25311,13 +25271,12 @@ define void @global_agent_atomic_fadd_noret_v2bf16__maybe_remote(ptr addrspace(1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -25330,7 +25289,6 @@ define void @global_agent_atomic_fadd_noret_v2bf16__maybe_remote(ptr addrspace(1
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB91_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
index f7cc0709109f9..ac223fd6030bd 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
@@ -15989,11 +15989,10 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -16111,7 +16110,6 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -16134,10 +16132,9 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -16149,7 +16146,6 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -16493,11 +16489,10 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -16615,7 +16610,6 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -16638,10 +16632,9 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -16653,7 +16646,6 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB55_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -16999,11 +16991,10 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -17121,7 +17112,6 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -17144,10 +17134,9 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -17159,7 +17148,6 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB56_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -17504,14 +17492,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -17627,7 +17613,6 @@ define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -17645,13 +17630,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -17664,7 +17648,6 @@ define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB57_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -17992,14 +17975,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -18115,7 +18096,6 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -18133,13 +18113,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -18152,7 +18131,6 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB58_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -18483,14 +18461,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -18606,7 +18582,6 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -18624,13 +18599,12 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -18643,7 +18617,6 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB59_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -18987,11 +18960,10 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -19111,7 +19083,6 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -19134,10 +19105,9 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -19149,7 +19119,6 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB60_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -19492,14 +19461,12 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -19617,7 +19584,6 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -19635,13 +19601,12 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -19654,7 +19619,6 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB61_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
index b81af1fc9233d..5653f85c67339 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
@@ -15989,11 +15989,10 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -16111,7 +16110,6 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -16134,10 +16132,9 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -16149,7 +16146,6 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -16493,11 +16489,10 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -16615,7 +16610,6 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -16638,10 +16632,9 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -16653,7 +16646,6 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB55_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -16999,11 +16991,10 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -17121,7 +17112,6 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -17144,10 +17134,9 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -17159,7 +17148,6 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB56_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -17504,14 +17492,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -17627,7 +17613,6 @@ define void @global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -17645,13 +17630,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -17664,7 +17648,6 @@ define void @global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB57_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -17992,14 +17975,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -18115,7 +18096,6 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -18133,13 +18113,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -18152,7 +18131,6 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB58_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -18483,14 +18461,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -18606,7 +18582,6 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -18624,13 +18599,12 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -18643,7 +18617,6 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB59_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -18987,11 +18960,10 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -19111,7 +19083,6 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -19134,10 +19105,9 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -19149,7 +19119,6 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB60_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -19492,14 +19461,12 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -19617,7 +19584,6 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -19635,13 +19601,12 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -19654,7 +19619,6 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB61_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
index b8762d13e1327..f0e16150c9e79 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
@@ -16350,11 +16350,10 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -16472,7 +16471,6 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -16495,10 +16493,9 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -16510,7 +16507,6 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB50_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -16854,11 +16850,10 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr addr
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -16976,7 +16971,6 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr addr
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -16999,10 +16993,9 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr addr
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -17014,7 +17007,6 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr addr
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB51_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -17360,11 +17352,10 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr addr
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -17482,7 +17473,6 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr addr
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -17505,10 +17495,9 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr addr
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -17520,7 +17509,6 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr addr
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB52_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -17865,14 +17853,12 @@ define void @global_agent_atomic_fsub_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -17988,7 +17974,6 @@ define void @global_agent_atomic_fsub_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -18006,13 +17991,12 @@ define void @global_agent_atomic_fsub_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -18025,7 +18009,6 @@ define void @global_agent_atomic_fsub_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB53_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -18353,14 +18336,12 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -18476,7 +18457,6 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -18494,13 +18474,12 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -18513,7 +18492,6 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -18844,14 +18822,12 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -18967,7 +18943,6 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -18985,13 +18960,12 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -19004,7 +18978,6 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB55_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -19348,11 +19321,10 @@ define <2 x bfloat> @global_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr add
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -19472,7 +19444,6 @@ define <2 x bfloat> @global_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr add
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -19495,10 +19466,9 @@ define <2 x bfloat> @global_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr add
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v5, 0xffff, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -19510,7 +19480,6 @@ define <2 x bfloat> @global_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr add
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB56_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -19853,14 +19822,12 @@ define void @global_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -19978,7 +19945,6 @@ define void @global_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -19996,13 +19962,12 @@ define void @global_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -20015,7 +19980,6 @@ define void @global_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB57_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll b/llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll
index 0dfeb3454dad5..7685f7384683b 100644
--- a/llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll
@@ -246,14 +246,12 @@ define <2 x bfloat> @v_uitofp_v2i1_to_v2bf16(<2 x i1> %num) {
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_uitofp_v2i1_to_v2bf16:
@@ -307,15 +305,13 @@ define <2 x bfloat> @v_uitofp_v2i1_to_v2bf16(<2 x i1> %num) {
; GFX12-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX12-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_uitofp_v2i1_to_v2bf16:
@@ -533,38 +529,36 @@ define <3 x bfloat> @v_uitofp_v3i1_to_v3bf16(<3 x i1> %num) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 1, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 1, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v6, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v3, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_uitofp_v3i1_to_v3bf16:
@@ -612,43 +606,41 @@ define <3 x bfloat> @v_uitofp_v3i1_to_v3bf16(<3 x i1> %num) {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 1, v1
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 1, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2
; GFX12-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
; GFX12-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v2
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v6, v2, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v7, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v3, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_uitofp_v3i1_to_v3bf16:
@@ -933,49 +925,44 @@ define <4 x bfloat> @v_uitofp_v4i1_to_v4bf16(<4 x i1> %num) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 1, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 1, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1.0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v6, 0, 1.0, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 1, v1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v9, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v9, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v2, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_uitofp_v4i1_to_v4bf16:
@@ -1033,57 +1020,51 @@ define <4 x bfloat> @v_uitofp_v4i1_to_v4bf16(<4 x i1> %num) {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 1, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 1, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 1, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v3
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1.0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, vcc_lo
-; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 1, v1
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v6, 0, 1.0, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v9, v1, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v9, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX12-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v2, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.h
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_uitofp_v4i1_to_v4bf16:
@@ -1588,14 +1569,12 @@ define <2 x bfloat> @v_sitofp_v2i1_to_v2bf16(<2 x i1> %num) {
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_sitofp_v2i1_to_v2bf16:
@@ -1649,15 +1628,13 @@ define <2 x bfloat> @v_sitofp_v2i1_to_v2bf16(<2 x i1> %num) {
; GFX12-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX12-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_sitofp_v2i1_to_v2bf16:
@@ -1877,38 +1854,36 @@ define <3 x bfloat> @v_sitofp_v3i1_to_v3bf16(<3 x i1> %num) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 1, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 1, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, -1.0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, -1.0, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, -1.0, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, -1.0, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v6, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v3, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_sitofp_v3i1_to_v3bf16:
@@ -1956,43 +1931,41 @@ define <3 x bfloat> @v_sitofp_v3i1_to_v3bf16(<3 x i1> %num) {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 1, v1
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 1, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2
; GFX12-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, -1.0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, -1.0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
; GFX12-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, -1.0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, -1.0, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v2
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v6, v2, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v7, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v3, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_sitofp_v3i1_to_v3bf16:
@@ -2280,49 +2253,44 @@ define <4 x bfloat> @v_sitofp_v4i1_to_v4bf16(<4 x i1> %num) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 1, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 1, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, -1.0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, -1.0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, -1.0, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v6, 0, -1.0, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 1, v1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, -1.0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v9, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v9, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v2, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_sitofp_v4i1_to_v4bf16:
@@ -2380,57 +2348,51 @@ define <4 x bfloat> @v_sitofp_v4i1_to_v4bf16(<4 x i1> %num) {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 1, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 1, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 1, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v3
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, -1.0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, -1.0, vcc_lo
-; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 1, v1
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, -1.0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v6, 0, -1.0, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v9, v1, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v9, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX12-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v2, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.h
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_sitofp_v4i1_to_v4bf16:
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll
index 2585167a6a98e..792d7db26d076 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll
@@ -751,19 +751,32 @@ define amdgpu_kernel void @v_insertelement_v2i16_0(ptr addrspace(1) %out, ptr ad
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
-; GFX11-LABEL: v_insertelement_v2i16_0:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v1, v0, s[2:3]
-; GFX11-NEXT: s_movk_i32 s2, 0x3e7
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v1, 0xffff, s2, v1
-; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-TRUE16-LABEL: v_insertelement_v2i16_0:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b32 v1, v0, s[2:3]
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x3e7
+; GFX11-TRUE16-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX11-TRUE16-NEXT: s_endpgm
+;
+; GFX11-FAKE16-LABEL: v_insertelement_v2i16_0:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_b32 v1, v0, s[2:3]
+; GFX11-FAKE16-NEXT: s_movk_i32 s2, 0x3e7
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_bfi_b32 v1, 0xffff, s2, v1
+; GFX11-FAKE16-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x() #1
%tid.ext = sext i32 %tid to i64
%in.gep = getelementptr inbounds <2 x i16>, ptr addrspace(1) %in, i64 %tid.ext
@@ -929,18 +942,31 @@ define amdgpu_kernel void @v_insertelement_v2i16_0_inlineimm(ptr addrspace(1) %o
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
-; GFX11-LABEL: v_insertelement_v2i16_0_inlineimm:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v1, v0, s[2:3]
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v1, 0xffff, 53, v1
-; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-TRUE16-LABEL: v_insertelement_v2i16_0_inlineimm:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b32 v1, v0, s[2:3]
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, 53
+; GFX11-TRUE16-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX11-TRUE16-NEXT: s_endpgm
+;
+; GFX11-FAKE16-LABEL: v_insertelement_v2i16_0_inlineimm:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_b32 v1, v0, s[2:3]
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_bfi_b32 v1, 0xffff, 53, v1
+; GFX11-FAKE16-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x() #1
%tid.ext = sext i32 %tid to i64
%in.gep = getelementptr inbounds <2 x i16>, ptr addrspace(1) %in, i64 %tid.ext
@@ -1190,19 +1216,32 @@ define amdgpu_kernel void @v_insertelement_v2f16_0(ptr addrspace(1) %out, ptr ad
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
-; GFX11-LABEL: v_insertelement_v2f16_0:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v1, v0, s[2:3]
-; GFX11-NEXT: s_movk_i32 s2, 0x4500
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v1, 0xffff, s2, v1
-; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-TRUE16-LABEL: v_insertelement_v2f16_0:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b32 v1, v0, s[2:3]
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x4500
+; GFX11-TRUE16-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX11-TRUE16-NEXT: s_endpgm
+;
+; GFX11-FAKE16-LABEL: v_insertelement_v2f16_0:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_b32 v1, v0, s[2:3]
+; GFX11-FAKE16-NEXT: s_movk_i32 s2, 0x4500
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_bfi_b32 v1, 0xffff, s2, v1
+; GFX11-FAKE16-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x() #1
%tid.ext = sext i32 %tid to i64
%in.gep = getelementptr inbounds <2 x half>, ptr addrspace(1) %in, i64 %tid.ext
@@ -1268,18 +1307,31 @@ define amdgpu_kernel void @v_insertelement_v2f16_0_inlineimm(ptr addrspace(1) %o
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
-; GFX11-LABEL: v_insertelement_v2f16_0_inlineimm:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v1, v0, s[2:3]
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v1, 0xffff, 53, v1
-; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-TRUE16-LABEL: v_insertelement_v2f16_0_inlineimm:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b32 v1, v0, s[2:3]
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, 53
+; GFX11-TRUE16-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX11-TRUE16-NEXT: s_endpgm
+;
+; GFX11-FAKE16-LABEL: v_insertelement_v2f16_0_inlineimm:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_b32 v1, v0, s[2:3]
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_bfi_b32 v1, 0xffff, 53, v1
+; GFX11-FAKE16-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX11-FAKE16-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x() #1
%tid.ext = sext i32 %tid to i64
%in.gep = getelementptr inbounds <2 x half>, ptr addrspace(1) %in, i64 %tid.ext
@@ -1821,19 +1873,33 @@ define amdgpu_kernel void @v_insertelement_v4f16_0(ptr addrspace(1) %out, ptr ad
; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; CI-NEXT: s_endpgm
;
-; GFX11-LABEL: v_insertelement_v4f16_0:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x30
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 3, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[0:1], v2, s[2:3]
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v0, 0xffff, s4, v0
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-TRUE16-LABEL: v_insertelement_v4f16_0:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-TRUE16-NEXT: s_load_b32 s4, s[4:5], 0x30
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v2, s[2:3]
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s4
+; GFX11-TRUE16-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-TRUE16-NEXT: s_endpgm
+;
+; GFX11-FAKE16-LABEL: v_insertelement_v4f16_0:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT: s_load_b32 s4, s[4:5], 0x30
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_b64 v[0:1], v2, s[2:3]
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, s4, v0
+; GFX11-FAKE16-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-FAKE16-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x() #1
%tid.ext = sext i32 %tid to i64
%in.gep = getelementptr inbounds <4 x half>, ptr addrspace(1) %in, i64 %tid.ext
@@ -2004,19 +2070,33 @@ define amdgpu_kernel void @v_insertelement_v4f16_2(ptr addrspace(1) %out, ptr ad
; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; CI-NEXT: s_endpgm
;
-; GFX11-LABEL: v_insertelement_v4f16_2:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x30
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 3, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[0:1], v2, s[2:3]
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v1, 0xffff, s4, v1
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-TRUE16-LABEL: v_insertelement_v4f16_2:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-TRUE16-NEXT: s_load_b32 s4, s[4:5], 0x30
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v2, s[2:3]
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s4
+; GFX11-TRUE16-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-TRUE16-NEXT: s_endpgm
+;
+; GFX11-FAKE16-LABEL: v_insertelement_v4f16_2:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT: s_load_b32 s4, s[4:5], 0x30
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_b64 v[0:1], v2, s[2:3]
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_bfi_b32 v1, 0xffff, s4, v1
+; GFX11-FAKE16-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-FAKE16-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x() #1
%tid.ext = sext i32 %tid to i64
%in.gep = getelementptr inbounds <4 x half>, ptr addrspace(1) %in, i64 %tid.ext
@@ -2187,19 +2267,33 @@ define amdgpu_kernel void @v_insertelement_v4i16_2(ptr addrspace(1) %out, ptr ad
; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; CI-NEXT: s_endpgm
;
-; GFX11-LABEL: v_insertelement_v4i16_2:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x10
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 3, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[0:1], v2, s[2:3]
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v1, 0xffff, s4, v1
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-TRUE16-LABEL: v_insertelement_v4i16_2:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-TRUE16-NEXT: s_load_b32 s4, s[4:5], 0x10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v2, s[2:3]
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s4
+; GFX11-TRUE16-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-TRUE16-NEXT: s_endpgm
+;
+; GFX11-FAKE16-LABEL: v_insertelement_v4i16_2:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT: s_load_b32 s4, s[4:5], 0x10
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 3, v0
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_b64 v[0:1], v2, s[2:3]
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_bfi_b32 v1, 0xffff, s4, v1
+; GFX11-FAKE16-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-FAKE16-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x() #1
%tid.ext = sext i32 %tid to i64
%in.gep = getelementptr inbounds <4 x i16>, ptr addrspace(1) %in, i64 %tid.ext
@@ -2592,19 +2686,33 @@ define amdgpu_kernel void @v_insertelement_v8i16_6(ptr addrspace(1) %out, ptr ad
; CI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; CI-NEXT: s_endpgm
;
-; GFX11-LABEL: v_insertelement_v8i16_6:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x10
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 4, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: global_load_b128 v[0:3], v4, s[2:3]
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v3, 0xffff, s4, v3
-; GFX11-NEXT: global_store_b128 v4, v[0:3], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-TRUE16-LABEL: v_insertelement_v8i16_6:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-TRUE16-NEXT: s_load_b32 s4, s[4:5], 0x10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 4, v0
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b128 v[0:3], v4, s[2:3]
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, s4
+; GFX11-TRUE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX11-TRUE16-NEXT: s_endpgm
+;
+; GFX11-FAKE16-LABEL: v_insertelement_v8i16_6:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT: s_load_b32 s4, s[4:5], 0x10
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 4, v0
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_b128 v[0:3], v4, s[2:3]
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_bfi_b32 v3, 0xffff, s4, v3
+; GFX11-FAKE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX11-FAKE16-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x() #1
%tid.ext = sext i32 %tid to i64
%in.gep = getelementptr inbounds <8 x i16>, ptr addrspace(1) %in, i64 %tid.ext
@@ -3087,24 +3195,42 @@ define amdgpu_kernel void @v_insertelement_v16i16_6(ptr addrspace(1) %out, ptr a
; CI-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
; CI-NEXT: s_endpgm
;
-; GFX11-LABEL: v_insertelement_v16i16_6:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x10
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v8, 5, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_load_b128 v[0:3], v8, s[2:3]
-; GFX11-NEXT: global_load_b128 v[4:7], v8, s[2:3] offset:16
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_bfi_b32 v3, 0xffff, s4, v3
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_store_b128 v8, v[4:7], s[0:1] offset:16
-; GFX11-NEXT: global_store_b128 v8, v[0:3], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-TRUE16-LABEL: v_insertelement_v16i16_6:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-TRUE16-NEXT: s_load_b32 s4, s[4:5], 0x10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 5, v0
+; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: global_load_b128 v[0:3], v8, s[2:3] offset:16
+; GFX11-TRUE16-NEXT: global_load_b128 v[4:7], v8, s[2:3]
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, s4
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: global_store_b128 v8, v[0:3], s[0:1] offset:16
+; GFX11-TRUE16-NEXT: global_store_b128 v8, v[4:7], s[0:1]
+; GFX11-TRUE16-NEXT: s_endpgm
+;
+; GFX11-FAKE16-LABEL: v_insertelement_v16i16_6:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT: s_load_b32 s4, s[4:5], 0x10
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 5, v0
+; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: global_load_b128 v[0:3], v8, s[2:3]
+; GFX11-FAKE16-NEXT: global_load_b128 v[4:7], v8, s[2:3] offset:16
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-FAKE16-NEXT: v_bfi_b32 v3, 0xffff, s4, v3
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_clause 0x1
+; GFX11-FAKE16-NEXT: global_store_b128 v8, v[4:7], s[0:1] offset:16
+; GFX11-FAKE16-NEXT: global_store_b128 v8, v[0:3], s[0:1]
+; GFX11-FAKE16-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x() #1
%tid.ext = sext i32 %tid to i64
%in.gep = getelementptr inbounds <16 x i16>, ptr addrspace(1) %in, i64 %tid.ext
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll
index 52f6dab902b3e..2ead741b454d3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll
@@ -436,9 +436,7 @@ define <2 x bfloat> @v_exp2_v2bf16(<2 x bfloat> %in) {
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1200-SDAG-FAKE16-LABEL: v_exp2_v2bf16:
@@ -552,9 +550,8 @@ define <2 x bfloat> @v_exp2_fabs_v2bf16(<2 x bfloat> %in) {
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX1200-SDAG-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1200-SDAG-FAKE16-LABEL: v_exp2_fabs_v2bf16:
@@ -659,29 +656,28 @@ define <2 x bfloat> @v_exp2_fneg_fabs_v2bf16(<2 x bfloat> %in) {
; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 0x42800000, s0
; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v1, v1, v3
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v2, v0
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0xffffffc0, s0
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0
+; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v2
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1200-SDAG-FAKE16-LABEL: v_exp2_fneg_fabs_v2bf16:
@@ -774,43 +770,43 @@ define <2 x bfloat> @v_exp2_fneg_v2bf16(<2 x bfloat> %in) {
; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
-; GFX1200-SDAG-TRUE16-NEXT: v_xor_b16 v1.h, 0x8000, v0.h
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: v_xor_b16 v1.h, 0x8000, v0.l
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v1
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0xffffffc0, vcc_lo
; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v2, v1, v2
-; GFX1200-SDAG-TRUE16-NEXT: v_xor_b16 v1.h, 0x8000, v0.l
+; GFX1200-SDAG-TRUE16-NEXT: v_xor_b16 v1.h, 0x8000, v0.h
; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0xc2fc0000, v1
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xf1ff
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 0x42800000, s0
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0xffffffc0, s0
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v1, v0
; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v1, v2
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0xffffffc0, s0
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0xffffffc0, vcc_lo
; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v1, v1, v3
-; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v2
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
-; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_2) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v1, v1, v2
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v3
+; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1200-SDAG-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1200-SDAG-FAKE16-LABEL: v_exp2_fneg_v2bf16:
@@ -929,9 +925,7 @@ define <2 x bfloat> @v_exp2_v2bf16_fast(<2 x bfloat> %in) {
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1200-SDAG-FAKE16-LABEL: v_exp2_v2bf16_fast:
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
index c1a32aafbc71e..8748aff42d65b 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
@@ -6834,7 +6834,6 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6857,12 +6856,11 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
@@ -6871,7 +6869,6 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB24_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -7191,7 +7188,6 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7214,12 +7210,11 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4 offset:65532
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4 offset:65532
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
@@ -7228,7 +7223,6 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB25_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -7549,7 +7543,6 @@ define void @local_atomic_fadd_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7570,11 +7563,10 @@ define void @local_atomic_fadd_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
@@ -7584,7 +7576,6 @@ define void @local_atomic_fadd_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB26_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -7893,7 +7884,6 @@ define void @local_atomic_fadd_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7914,11 +7904,10 @@ define void @local_atomic_fadd_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3 offset:65532
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3 offset:65532
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
@@ -7928,7 +7917,6 @@ define void @local_atomic_fadd_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
index 739e86d1928b1..d6b7d8ffaf1c5 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
@@ -6651,13 +6651,12 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
@@ -6771,7 +6770,6 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6794,12 +6792,11 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
@@ -6808,7 +6805,6 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB24_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -7133,13 +7129,12 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4 offset:65532
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4 offset:65532
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
@@ -7253,7 +7248,6 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7276,12 +7270,11 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4 offset:65532
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4 offset:65532
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
@@ -7290,7 +7283,6 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB25_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -7615,11 +7607,9 @@ define void @local_atomic_fmax_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
@@ -7730,7 +7720,6 @@ define void @local_atomic_fmax_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7751,11 +7740,10 @@ define void @local_atomic_fmax_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
@@ -7765,7 +7753,6 @@ define void @local_atomic_fmax_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB26_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -8078,11 +8065,9 @@ define void @local_atomic_fmax_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3 offset:65532
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3 offset:65532
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
@@ -8193,7 +8178,6 @@ define void @local_atomic_fmax_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8214,11 +8198,10 @@ define void @local_atomic_fmax_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3 offset:65532
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3 offset:65532
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
@@ -8228,7 +8211,6 @@ define void @local_atomic_fmax_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
index 6da80262951e5..11ed43d737634 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
@@ -6651,13 +6651,12 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
@@ -6771,7 +6770,6 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6794,12 +6792,11 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
@@ -6808,7 +6805,6 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB24_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -7133,13 +7129,12 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4 offset:65532
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4 offset:65532
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
@@ -7253,7 +7248,6 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7276,12 +7270,11 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4 offset:65532
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4 offset:65532
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
@@ -7290,7 +7283,6 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB25_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -7615,11 +7607,9 @@ define void @local_atomic_fmin_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
@@ -7730,7 +7720,6 @@ define void @local_atomic_fmin_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7751,11 +7740,10 @@ define void @local_atomic_fmin_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
@@ -7765,7 +7753,6 @@ define void @local_atomic_fmin_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB26_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -8078,11 +8065,9 @@ define void @local_atomic_fmin_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3 offset:65532
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3 offset:65532
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
@@ -8193,7 +8178,6 @@ define void @local_atomic_fmin_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8214,11 +8198,10 @@ define void @local_atomic_fmin_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3 offset:65532
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3 offset:65532
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
@@ -8228,7 +8211,6 @@ define void @local_atomic_fmin_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
index 786989cc9fb57..d74338caba1cd 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
@@ -7419,13 +7419,12 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
@@ -7539,7 +7538,6 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7562,12 +7560,11 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
@@ -7576,7 +7573,6 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB24_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -7901,13 +7897,12 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4 offset:65532
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4 offset:65532
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
@@ -8021,7 +8016,6 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8044,12 +8038,11 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0xffff, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4 offset:65532
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4 offset:65532
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
@@ -8058,7 +8051,6 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB25_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -8383,11 +8375,9 @@ define void @local_atomic_fsub_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
@@ -8498,7 +8488,6 @@ define void @local_atomic_fsub_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8519,11 +8508,10 @@ define void @local_atomic_fsub_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
@@ -8533,7 +8521,6 @@ define void @local_atomic_fsub_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB26_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -8846,11 +8833,9 @@ define void @local_atomic_fsub_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_wait_alu 0xfffd
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3 offset:65532
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3 offset:65532
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
@@ -8961,7 +8946,6 @@ define void @local_atomic_fsub_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8982,11 +8966,10 @@ define void @local_atomic_fsub_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_bfi_b32 v4, 0xffff, v4, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3 offset:65532
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3 offset:65532
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
@@ -8996,7 +8979,6 @@ define void @local_atomic_fsub_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/vector_rebroadcast.ll b/llvm/test/CodeGen/AMDGPU/vector_rebroadcast.ll
index 587f5d05d358b..07e9325095017 100644
--- a/llvm/test/CodeGen/AMDGPU/vector_rebroadcast.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector_rebroadcast.ll
@@ -1079,9 +1079,7 @@ define <2 x bfloat> @shuffle_v2bf16_rebroadcast(ptr addrspace(1) %arg0) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v2bf16_rebroadcast:
@@ -1120,12 +1118,10 @@ define <3 x bfloat> @shuffle_v3bf16_rebroadcast(ptr addrspace(1) %arg0) {
; GFX11-TRUE16-LABEL: shuffle_v3bf16_rebroadcast:
; GFX11-TRUE16: ; %bb.0: ; %entry
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v3bf16_rebroadcast:
@@ -1167,9 +1163,8 @@ define <4 x bfloat> @shuffle_v4bf16_rebroadcast(ptr addrspace(1) %arg0) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1215,9 +1210,8 @@ define <6 x bfloat> @shuffle_v6bf16_rebroadcast(ptr addrspace(1) %arg0) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1267,9 +1261,8 @@ define <8 x bfloat> @shuffle_v8bf16_rebroadcast(ptr addrspace(1) %arg0) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
@@ -1329,9 +1322,8 @@ define <16 x bfloat> @shuffle_v16bf16_rebroadcast(ptr addrspace(1) %arg0) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
@@ -1415,9 +1407,8 @@ define <32 x bfloat> @shuffle_v32bf16_rebroadcast(ptr addrspace(1) %arg0) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
@@ -1487,9 +1478,7 @@ define <2 x half> @shuffle_v2f16_rebroadcast(ptr addrspace(1) %arg0) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v2f16_rebroadcast:
@@ -1528,12 +1517,10 @@ define <3 x half> @shuffle_v3f16_rebroadcast(ptr addrspace(1) %arg0) {
; GFX11-TRUE16-LABEL: shuffle_v3f16_rebroadcast:
; GFX11-TRUE16: ; %bb.0: ; %entry
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v3f16_rebroadcast:
@@ -1575,9 +1562,8 @@ define <4 x half> @shuffle_v4f16_rebroadcast(ptr addrspace(1) %arg0) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1623,9 +1609,8 @@ define <6 x half> @shuffle_v6f16_rebroadcast(ptr addrspace(1) %arg0) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1675,9 +1660,8 @@ define <8 x half> @shuffle_v8f16_rebroadcast(ptr addrspace(1) %arg0) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
@@ -1737,9 +1721,8 @@ define <16 x half> @shuffle_v16f16_rebroadcast(ptr addrspace(1) %arg0) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
@@ -1823,9 +1806,8 @@ define <32 x half> @shuffle_v32f16_rebroadcast(ptr addrspace(1) %arg0) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
diff --git a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
index fe7def8a69278..7e4dd81438948 100644
--- a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
@@ -331,8 +331,7 @@ define <4 x half> @shuffle_v4f16_35u5(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4f16_35u5:
@@ -390,12 +389,9 @@ define <4 x half> @shuffle_v4f16_357u(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[2:3], off
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4f16_357u:
@@ -1225,13 +1221,15 @@ define <4 x half> @shuffle_v4f16_5734(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-LABEL: shuffle_v4f16_5734:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
-; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4f16_5734:
@@ -1482,10 +1480,11 @@ define <4 x half> @shuffle_v4f16_1100(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-LABEL: shuffle_v4f16_1100:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b64 v[1:2], v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4f16_1100:
@@ -1538,13 +1537,11 @@ define <4 x half> @shuffle_v4f16_6161(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-LABEL: shuffle_v4f16_6161:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v2, v[2:3], off offset:4
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b32 v0, v[2:3], off offset:4
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1597,8 +1594,7 @@ define <4 x half> @shuffle_v4f16_2333(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4f16_2333:
@@ -1647,8 +1643,7 @@ define <4 x half> @shuffle_v4f16_6667(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4f16_6667:
@@ -2318,13 +2313,10 @@ define <2 x half> @hi16bits_v2f16(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
; GFX11-TRUE16-LABEL: hi16bits_v2f16:
; GFX11-TRUE16: ; %bb.0: ; %entry
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
-; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b32 v0, v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: hi16bits_v2f16:
@@ -2373,14 +2365,23 @@ define <2 x half> @low16hi16bits_v2f16(ptr addrspace(1) %x0, ptr addrspace(1) %x
; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v4, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: low16hi16bits_v2f16:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v0, v[0:1], off
-; GFX11-NEXT: global_load_b32 v1, v[2:3], off
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: low16hi16bits_v2f16:
+; GFX11-TRUE16: ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b32 v2, v[2:3], off
+; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.h
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: low16hi16bits_v2f16:
+; GFX11-FAKE16: ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-FAKE16-NEXT: global_load_b32 v1, v[2:3], off
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%0 = load <2 x half>, ptr addrspace(1) %x0, align 4
%1 = load <2 x half>, ptr addrspace(1) %x1, align 4
@@ -2520,14 +2521,23 @@ define <2 x i16> @i16_low16hi16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1)
; GFX10-NEXT: v_bfi_b32 v0, 0xffff, v4, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: i16_low16hi16bits:
-; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v0, v[0:1], off
-; GFX11-NEXT: global_load_b32 v1, v[2:3], off
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: i16_low16hi16bits:
+; GFX11-TRUE16: ; %bb.0: ; %entry
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b32 v2, v[2:3], off
+; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.h
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: i16_low16hi16bits:
+; GFX11-FAKE16: ; %bb.0: ; %entry
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-FAKE16-NEXT: global_load_b32 v1, v[2:3], off
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
entry:
%0 = load <2 x i16>, ptr addrspace(1) %x0, align 4
%1 = load <2 x i16>, ptr addrspace(1) %x1, align 4
@@ -3617,8 +3627,7 @@ define <4 x bfloat> @shuffle_v4bf16_35u5(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4bf16_35u5:
@@ -3676,12 +3685,9 @@ define <4 x bfloat> @shuffle_v4bf16_357u(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[2:3], off
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4bf16_357u:
@@ -4511,13 +4517,15 @@ define <4 x bfloat> @shuffle_v4bf16_5734(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-LABEL: shuffle_v4bf16_5734:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
-; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4bf16_5734:
@@ -4671,10 +4679,11 @@ define <4 x bfloat> @shuffle_v4bf16_1100(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-LABEL: shuffle_v4bf16_1100:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b64 v[1:2], v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4bf16_1100:
@@ -4727,13 +4736,11 @@ define <4 x bfloat> @shuffle_v4bf16_6161(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-LABEL: shuffle_v4bf16_6161:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v2, v[2:3], off offset:4
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b32 v0, v[2:3], off offset:4
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v1, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4786,8 +4793,7 @@ define <4 x bfloat> @shuffle_v4bf16_2333(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4bf16_2333:
@@ -4836,8 +4842,7 @@ define <4 x bfloat> @shuffle_v4bf16_6667(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v1, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4bf16_6667:
@@ -5533,13 +5538,9 @@ define amdgpu_kernel void @fma_shuffle_v2bf16(ptr addrspace(1) nocapture readonl
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v5.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v3, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0xffff, v2, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.h
; GFX11-TRUE16-NEXT: global_store_b64 v6, v[0:1], s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
;
@@ -5817,13 +5818,10 @@ define <2 x bfloat> @hi16bits_v2bf16(ptr addrspace(1) %x0, ptr addrspace(1) %x1)
; GFX11-TRUE16-LABEL: hi16bits_v2bf16:
; GFX11-TRUE16: ; %bb.0: ; %entry
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
-; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b32 v0, v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: hi16bits_v2bf16:
@@ -5875,10 +5873,10 @@ define <2 x bfloat> @low16hi16bits_v2bf16(ptr addrspace(1) %x0, ptr addrspace(1)
; GFX11-TRUE16-LABEL: low16hi16bits_v2bf16:
; GFX11-TRUE16: ; %bb.0: ; %entry
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: global_load_b32 v2, v[2:3], off
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
-; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: low16hi16bits_v2bf16:
More information about the llvm-commits
mailing list