[llvm] AMDGPU: Fix true16 build_vector (0, x) pattern using a 16-bit shift operand (PR #230383)
Matt Arsenault via llvm-commits
llvm-commits at lists.llvm.org
Sun Oct 11 03:29:07 PDT 2026
https://github.com/arsenm updated https://github.com/llvm/llvm-project/pull/230383
>From 34c4aa6f9b1e28bf7e84c1c984a6840726094e05 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Thu, 8 Oct 2026 21:31:06 +0200
Subject: [PATCH] AMDGPU: Fix true16 build_vector (0, x) pattern using a 16-bit
shift operand
The real true16 pattern for (build_vector 0, VGPR_16:$x) fed the 16-bit
register directly to V_LSHLREV_B32, which takes a 32-bit operand. Widen
it with a REG_SEQUENCE first. This avoids redundant 16-bit moves in
SelectionDAG, and fixes a GlobalISel selection failure when the 16-bit
input is a G_TRUNC of a 32-bit value, as the shift's operand class
constrained the trunc result to vgpr_32.
I also don't know why this pattern is overcomplicating this. I would expect
true16 to literally translate build_vector to reg_sequence plus a materialize
of the 0.
Co-authored-by: Claude Opus 5.5 <noreply at anthropic.com>
---
llvm/lib/Target/AMDGPU/SIInstructions.td | 3 +-
.../inst-select-build-vector-true16.mir | 52 +
.../CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll | 739 ++-
.../CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll | 64 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll | 435 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll | 431 +-
llvm/test/CodeGen/AMDGPU/bf16.ll | 98 +-
.../buffer-fat-pointer-atomicrmw-fadd.ll | 1161 ++--
.../buffer-fat-pointer-atomicrmw-fmax.ll | 1207 +++--
.../buffer-fat-pointer-atomicrmw-fmin.ll | 1207 +++--
llvm/test/CodeGen/AMDGPU/fabs.bf16.ll | 49 +-
.../CodeGen/AMDGPU/flat-atomicrmw-fadd.ll | 1610 ++++--
.../CodeGen/AMDGPU/flat-atomicrmw-fmax.ll | 1560 ++++--
.../CodeGen/AMDGPU/flat-atomicrmw-fmin.ll | 1560 ++++--
.../CodeGen/AMDGPU/flat-atomicrmw-fsub.ll | 1560 ++++--
.../float-to-arbitrary-fp-fp8-f16-hw.ll | 30 +-
.../AMDGPU/float-to-arbitrary-fp-fp8-hw.ll | 343 +-
.../AMDGPU/float-to-arbitrary-fp-widen.ll | 2949 +++++-----
llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll | 212 +-
llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll | 26 +-
llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll | 212 +-
llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll | 146 +-
llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll | 170 +-
.../CodeGen/AMDGPU/global-atomicrmw-fadd.ll | 1560 ++++--
.../CodeGen/AMDGPU/global-atomicrmw-fmax.ll | 1560 ++++--
.../CodeGen/AMDGPU/global-atomicrmw-fmin.ll | 1560 ++++--
.../CodeGen/AMDGPU/global-atomicrmw-fsub.ll | 1560 ++++--
.../isel-amdgpu-cs-chain-preserve-cc.ll | 128 +-
llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll | 97 +-
llvm/test/CodeGen/AMDGPU/llvm.log2.ll | 43 +-
llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll | 4762 ++++++++---------
llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll | 4762 ++++++++---------
32 files changed, 18050 insertions(+), 13806 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-build-vector-true16.mir
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index ba02d0192a7ac..c881cfdeebdb9 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -3999,7 +3999,8 @@ def : GCNPat <
let True16Predicate = UseRealTrue16Insts in {
def : GCNPat <
(v2i16 (DivergentBinFrag<build_vector> (i16 0), (i16 VGPR_16:$src1))),
- (v2i16 (V_LSHLREV_B32_e64 (i16 16), VGPR_16:$src1))
+ (v2i16 (V_LSHLREV_B32_e64 (i32 16),
+ (i32 (REG_SEQUENCE VGPR_32, VGPR_16:$src1, lo16, (i16 (IMPLICIT_DEF)), hi16))))
>;
def : GCNPat <
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-build-vector-true16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-build-vector-true16.mir
new file mode 100644
index 0000000000000..51acb294d3fc5
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-build-vector-true16.mir
@@ -0,0 +1,52 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu11.00 -mattr=+real-true16 -run-pass=instruction-select -o - %s | FileCheck %s
+
+---
+name: test_build_vector_v_v2s16_zero_lo_trunc_hi
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+ bb.0:
+ liveins: $vgpr0
+
+ ; CHECK-LABEL: name: test_build_vector_v_v2s16_zero_lo_trunc_hi
+ ; CHECK: liveins: $vgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
+ ; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY1]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; CHECK-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[REG_SEQUENCE]], implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[V_LSHLREV_B32_e64_]]
+ %0:vgpr(i32) = COPY $vgpr0
+ %1:vgpr(i16) = G_TRUNC %0
+ %2:sgpr(i16) = G_CONSTANT i16 0
+ %3:vgpr(i16) = COPY %2
+ %4:vgpr(<2 x i16>) = G_BUILD_VECTOR %3, %1
+ S_ENDPGM 0, implicit %4
+...
+
+---
+name: test_build_vector_s_v2s16_zero_lo_trunc_hi
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+ bb.0:
+ liveins: $sgpr0
+
+ ; CHECK-LABEL: name: test_build_vector_s_v2s16_zero_lo_trunc_hi
+ ; CHECK: liveins: $sgpr0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY]], 16, implicit-def dead $scc
+ ; CHECK-NEXT: S_ENDPGM 0, implicit [[S_LSHL_B32_]]
+ %0:sgpr(i32) = COPY $sgpr0
+ %1:sgpr(i16) = G_TRUNC %0
+ %2:sgpr(i16) = G_CONSTANT i16 0
+ %3:sgpr(<2 x i16>) = G_BUILD_VECTOR %2, %1
+ S_ENDPGM 0, implicit %3
+...
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
index 4b30ed2dfe6db..f29518ca5861a 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
@@ -2492,76 +2492,74 @@ define <4 x i32> @bitcast_v8bf16_to_v4i32(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v10, v13 :: v_dual_and_b32 v5, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v2, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v6, v7 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v6, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v13, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v11, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v14, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v3, v4 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.h
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v8 :: v_dual_lshlrev_b32 v8, 16, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v8
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v11, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v3, v9 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v3, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v11, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v2, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.h
; GFX11-TRUE16-NEXT: .LBB22_4: ; %end
@@ -6721,76 +6719,74 @@ define <4 x float> @bitcast_v8bf16_to_v4f32(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v10, v13 :: v_dual_and_b32 v5, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v2, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v6, v7 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v6, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v13, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v11, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v14, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v3, v4 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.h
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v8 :: v_dual_lshlrev_b32 v8, 16, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v8
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v11, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v3, v9 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v3, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v11, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v2, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.h
; GFX11-TRUE16-NEXT: .LBB46_4: ; %end
@@ -10607,76 +10603,74 @@ define <2 x i64> @bitcast_v8bf16_to_v2i64(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB66_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v10, v13 :: v_dual_and_b32 v5, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v2, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v6, v7 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v6, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v13, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v11, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v14, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v3, v4 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.h
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v8 :: v_dual_lshlrev_b32 v8, 16, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v8
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v11, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v3, v9 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v3, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v11, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v2, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.h
; GFX11-TRUE16-NEXT: .LBB66_4: ; %end
@@ -14130,76 +14124,74 @@ define <2 x double> @bitcast_v8bf16_to_v2f64(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v10, v13 :: v_dual_and_b32 v5, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v2, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v6, v7 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v6, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v13, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v11, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v14, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v3, v4 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.h
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v8 :: v_dual_lshlrev_b32 v8, 16, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v8
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v11, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v3, v9 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v3, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v11, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v2, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.h
; GFX11-TRUE16-NEXT: .LBB82_4: ; %end
@@ -17357,76 +17349,69 @@ define <8 x i16> @bitcast_v8bf16_to_v8i16(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB94_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v6, 16, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v5, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v4, 0x40c00000, v4
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v7, v8 :: v_dual_and_b32 v1, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v12, v5, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v9, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v11, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v13, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_lshlrev_b32 v7, 16, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v10
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v9, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v6
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v9, v10, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v10, v11, v7, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v9, v12 :: v_dual_and_b32 v3, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v9, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v13, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.h
; GFX11-TRUE16-NEXT: .LBB94_2: ; %end
@@ -20144,90 +20129,91 @@ define <8 x half> @bitcast_v8bf16_to_v8f16(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8bf16_to_v8f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v4
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, v3 :: v_dual_mov_b32 v6, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB102_3
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB102_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB102_4
+; GFX11-TRUE16-NEXT: .LBB102_2: ; %end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-NEXT: .LBB102_3: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5
+; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB102_2
+; GFX11-TRUE16-NEXT: .LBB102_4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_add_f32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v1, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v7, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v4, v8 :: v_dual_add_f32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.l
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v11, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v13, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v7 :: v_dual_add_f32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_cndmask_b32 v0, v5, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v10, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v9, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.h
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v8
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v10, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v10, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v7
; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v3, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v13, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v10, v14, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v8, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v10.h
-; GFX11-TRUE16-NEXT: .LBB102_4: ; %end
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v5.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v10, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v13, v8, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v9, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.h
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v6 :: v_dual_mov_b32 v3, v7
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8bf16_to_v8f16:
@@ -22807,82 +22793,75 @@ define <16 x i8> @bitcast_v8bf16_to_v16i8(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v6, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v6
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v14, v4, v6, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v7 :: v_dual_lshlrev_b32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v14, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v9, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_add_f32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v13, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v11, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v14, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v5, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v8, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v2
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v3, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v7, v9, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v6.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v0, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11-TRUE16-NEXT: v_add3_u32 v0, v11, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[22:23], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v6.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v7, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v21, v3, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[22:23], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v21, v5, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v12.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 8, v16
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v0, v3, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v8.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v21
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v21
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[20:21]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v20
; GFX11-TRUE16-NEXT: .LBB108_4: ; %end
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
index 4700a3abc0e54..7cd976d99f8de 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
@@ -148,34 +148,32 @@ define <3 x half> @bitcast_v3bf16_to_v3f16(<3 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB0_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v7, v3, v7 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0x7fc0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
; GFX11-TRUE16-NEXT: .LBB0_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
@@ -820,34 +818,32 @@ define <3 x i16> @bitcast_v3bf16_to_v3i16(<3 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB4_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v3, v7 :: v_dual_mov_b32 v3, 0x7fc0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v4, v5 :: v_dual_and_b32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v1, 0x5040302
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
; GFX11-TRUE16-NEXT: .LBB4_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
index 937ec4e5c1d13..1b7575ea1c6c7 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
@@ -1965,42 +1965,38 @@ define i64 @bitcast_v4bf16_to_i64(<4 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v9, v11 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
; GFX11-TRUE16-NEXT: .LBB22_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -4857,42 +4853,38 @@ define double @bitcast_v4bf16_to_f64(<4 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v9, v11 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
; GFX11-TRUE16-NEXT: .LBB46_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -7503,42 +7495,38 @@ define <2 x i32> @bitcast_v4bf16_to_v2i32(<4 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB66_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v9, v11 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
; GFX11-TRUE16-NEXT: .LBB66_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -9854,42 +9842,38 @@ define <2 x float> @bitcast_v4bf16_to_v2f32(<4 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v9, v11 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
; GFX11-TRUE16-NEXT: .LBB82_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -11924,42 +11908,38 @@ define <4 x i16> @bitcast_v4bf16_to_v4i16(<4 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB94_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_add_f32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v9, v11 :: v_dual_add_f32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
; GFX11-TRUE16-NEXT: .LBB94_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -13655,44 +13635,41 @@ define <4 x half> @bitcast_v4bf16_to_v4f16(<4 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB102_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v9, v11 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v9.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
; GFX11-TRUE16-NEXT: .LBB102_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -15256,43 +15233,39 @@ define <8 x i8> @bitcast_v4bf16_to_v8i8(<4 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v4, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v10, v5, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v2, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v9, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v11, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v6, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v12, v0, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v8
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
index ec65609a9dcdc..f9d07f4800fc0 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
@@ -1878,56 +1878,53 @@ define <3 x i32> @bitcast_v6bf16_to_v3i32(<6 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB10_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v5 :: v_dual_lshlrev_b32 v0, 16, v0
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v8, v11 :: v_dual_and_b32 v2, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v0, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v8, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v7, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v10, v7, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.h
; GFX11-TRUE16-NEXT: .LBB10_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
@@ -4859,56 +4856,53 @@ define <3 x float> @bitcast_v6bf16_to_v3f32(<6 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB26_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v5 :: v_dual_lshlrev_b32 v0, 16, v0
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v8, v11 :: v_dual_and_b32 v2, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v0, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v8, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v7, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v10, v7, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.h
; GFX11-TRUE16-NEXT: .LBB26_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
@@ -7316,67 +7310,71 @@ define <12 x i8> @bitcast_v6bf16_to_v12i8(<6 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB38_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v17, 0x7fc07fc0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v4 :: v_dual_add_f32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v17, 0x7fc07fc0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v12, v4, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v4, v8 :: v_dual_add_f32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v3, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v12, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v1, v3, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v0, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v1, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v9, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v10, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v3.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v9, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v0, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v3.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v8.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[12:13], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[14:15], 24, v[16:17]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v10
; GFX11-TRUE16-NEXT: .LBB38_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v12.l
@@ -10748,79 +10746,76 @@ define <6 x half> @bitcast_v6bf16_to_v6f16(<6 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v6bf16_to_v6f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v5, v1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_vgpr4_vgpr5_vgpr6
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB48_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB48_4
-; GFX11-TRUE16-NEXT: .LBB48_2: ; %end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB48_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v6 :: v_dual_mov_b32 v3, v7
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v5, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB48_2
-; GFX11-TRUE16-NEXT: .LBB48_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v10, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v0, v5, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB48_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v10, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.h
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v8, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v9, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v5.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v10, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v10
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v10, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_cndmask_b32 v1, v1, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v11, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
+; GFX11-TRUE16-NEXT: .LBB48_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v3 :: v_dual_mov_b32 v1, v4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v6bf16_to_v6f16:
@@ -11780,55 +11775,53 @@ define <6 x i16> @bitcast_v6bf16_to_v6i16(<6 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB52_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v1, 0x40c00000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_add_f32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v9, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v10, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v2, 0x40c00000, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v9, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v9, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v5.h
; GFX11-TRUE16-NEXT: .LBB52_2: ; %end
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index bb62ee924c5ad..25158e036e29e 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -13473,7 +13473,7 @@ define bfloat @v_fadd_bf16_fpimm_0(bfloat %arg0) #0 {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_pk_add_bf16 v0, v0, 1.0
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, 1.0 op_sel:[0,1]
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%add = fadd bfloat %arg0, 1.0
ret bfloat %add
@@ -29636,12 +29636,12 @@ define bfloat @v_log2_bf16(bfloat %a) #0 {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 32, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo
+; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
-; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_log_f32_e32 v0, v0
; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1
@@ -30122,15 +30122,15 @@ define bfloat @v_exp_bf16(bfloat %a) #0 {
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0
-; GFX11TRUE16-NEXT: v_rndne_f32_e32 v2, v1
-; GFX11TRUE16-NEXT: v_fma_f32 v3, v0, 0x3fb8aa3b, -v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_sub_f32_e32 v1, v1, v2
-; GFX11TRUE16-NEXT: v_fmamk_f32 v3, v0, 0x32a5705f, v3
-; GFX11TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX11TRUE16-NEXT: v_fma_f32 v2, v0, 0x3fb8aa3b, -v1
+; GFX11TRUE16-NEXT: v_rndne_f32_e32 v3, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_sub_f32_e32 v1, v1, v3
; GFX11TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc2ce8ed0, v0
+; GFX11TRUE16-NEXT: v_fmamk_f32 v2, v0, 0x32a5705f, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX11TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v3
; GFX11TRUE16-NEXT: v_exp_f32_e32 v1, v1
; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v2
@@ -30360,12 +30360,12 @@ define bfloat @v_exp2_bf16(bfloat %a) #0 {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42800000, vcc_lo
+; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo
-; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_exp_f32_e32 v0, v0
; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
@@ -30590,15 +30590,15 @@ define bfloat @v_exp10_bf16(bfloat %a) #0 {
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0
-; GFX11TRUE16-NEXT: v_rndne_f32_e32 v2, v1
-; GFX11TRUE16-NEXT: v_fma_f32 v3, v0, 0x40549a78, -v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_sub_f32_e32 v1, v1, v2
-; GFX11TRUE16-NEXT: v_fmamk_f32 v3, v0, 0x33979a37, v3
-; GFX11TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX11TRUE16-NEXT: v_fma_f32 v2, v0, 0x40549a78, -v1
+; GFX11TRUE16-NEXT: v_rndne_f32_e32 v3, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_sub_f32_e32 v1, v1, v3
; GFX11TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc23369f4, v0
+; GFX11TRUE16-NEXT: v_fmamk_f32 v2, v0, 0x33979a37, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX11TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v3
; GFX11TRUE16-NEXT: v_exp_f32_e32 v1, v1
; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v2
@@ -31315,21 +31315,39 @@ define bfloat @v_round_bf16(bfloat %a) #0 {
; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250-LABEL: v_round_bf16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_trunc_f32_e32 v2, v1
-; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, -v2, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,1]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_cmp_ge_f32_e64 s0, |v0|, 0.5
-; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_bfi_b32 v0, 0x7fffffff, v0, v1
-; GFX1250-NEXT: v_fma_mixlo_bf16 v0, v2, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,0]
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250TRUE16-LABEL: v_round_bf16:
+; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX1250TRUE16-NEXT: v_trunc_f32_e32 v2, v1
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v0, -v2, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,1]
+; GFX1250TRUE16-NEXT: v_cmp_ge_f32_e64 s0, |v0|, 0.5
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s0
+; GFX1250TRUE16-NEXT: v_bfi_b32 v0, 0x7fffffff, v0, v1
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_fma_mixlo_bf16 v0, v2, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,0]
+; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+;
+; GFX1250FAKE16-LABEL: v_round_bf16:
+; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_trunc_f32_e32 v2, v1
+; GFX1250FAKE16-NEXT: v_fma_mix_f32_bf16 v0, -v2, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,1]
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_cmp_ge_f32_e64 s0, |v0|, 0.5
+; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s0
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_bfi_b32 v0, 0x7fffffff, v0, v1
+; GFX1250FAKE16-NEXT: v_fma_mixlo_bf16 v0, v2, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,0]
+; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
%op = call bfloat @llvm.round.bf16(bfloat %a)
ret bfloat %op
}
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index e8ed1993b8978..34c86d6a54a5a 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -4966,63 +4966,120 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; --------------------------------------------------------------------
define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_b32 s4, s16, -4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v4, s4
-; GFX12-NEXT: s_and_b32 s4, s16, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
-; GFX12-NEXT: s_not_b32 s6, s5
-; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB16_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: v_add_f32_e32 v0, v0, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB16_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX12-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v0, v0, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB16_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX12-FAKE16-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v0, v0, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB16_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -5067,57 +5124,108 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_addk_i32 s16, 0x200
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-NEXT: s_and_b32 s4, s16, -4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v4, s4
-; GFX11-NEXT: s_and_b32 s4, s16, 3
-; GFX11-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
-; GFX11-NEXT: s_not_b32 s6, s5
-; GFX11-NEXT: s_mov_b32 s5, 0
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB16_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_add_f32_e32 v0, v0, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX11-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB16_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB16_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB16_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -5372,62 +5480,118 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
}
define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_b32 s4, s16, -4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-NEXT: s_and_b32 s4, s16, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
-; GFX12-NEXT: s_not_b32 s6, s5
-; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB17_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: v_add_f32_e32 v0, v0, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, v4
-; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB17_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX12-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB17_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX12-FAKE16-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB17_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -5471,56 +5635,106 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_addk_i32 s16, 0x200
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-NEXT: s_and_b32 s4, s16, -4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
-; GFX11-NEXT: s_and_b32 s4, s16, 3
-; GFX11-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
-; GFX11-NEXT: s_not_b32 s6, s5
-; GFX11-NEXT: s_mov_b32 s5, 0
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB17_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_add_f32_e32 v0, v0, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX11-NEXT: v_mov_b32_e32 v1, v4
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB17_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB17_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB17_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -5769,94 +5983,185 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
}
define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX12-NEXT: s_mov_b32 s4, exec_lo
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s5, s4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_and_b32_e32 v6, 3, v4
-; GFX12-NEXT: v_and_b32_e32 v8, -4, v4
-; GFX12-NEXT: v_lshlrev_b32_e32 v7, 3, v6
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
-; GFX12-NEXT: v_not_b32_e32 v9, v6
-; GFX12-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_readfirstlane_b32 s0, v0
-; GFX12-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-NEXT: v_readfirstlane_b32 s2, v2
-; GFX12-NEXT: v_readfirstlane_b32 s3, v3
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
-; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB18_1
-; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s4
-; GFX12-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB18_3: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Loop Header: Depth=1
-; GFX12-NEXT: ; Child Loop BB18_4 Depth 2
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX12-NEXT: s_mov_b32 s5, exec_lo
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s6, s5
-; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v4, v4, v10
-; GFX12-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v11, 0x400000, v4
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX12-NEXT: v_lshlrev_b32_e32 v4, v7, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v9, v4
-; GFX12-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
-; GFX12-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
-; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-NEXT: v_readfirstlane_b32 s0, v0
-; GFX12-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-NEXT: v_readfirstlane_b32 s2, v2
-; GFX12-NEXT: v_readfirstlane_b32 s3, v3
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX12-NEXT: s_cbranch_execnz .LBB18_4
-; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX12-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
-; GFX12-NEXT: v_mov_b32_e32 v6, v4
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB18_3
-; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v7, v4
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v9, v6
+; GFX12-TRUE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_1
+; GFX12-TRUE16-NEXT: ; %bb.2:
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v4
+; GFX12-TRUE16-NEXT: .LBB18_3: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: ; Child Loop BB18_4 Depth 2
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v4, v4, v10
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
+; GFX12-TRUE16-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
+; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_4
+; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v4
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_3
+; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v9, v6
+; GFX12-FAKE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_1
+; GFX12-FAKE16-NEXT: ; %bb.2:
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-FAKE16-NEXT: .LBB18_3: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: ; Child Loop BB18_4 Depth 2
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v4, v4, v10
+; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
+; GFX12-FAKE16-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
+; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_4
+; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v4
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_3
+; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -5933,89 +6238,175 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amd
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v8, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_mov_b32 s6, s5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_b32_e32 v6, 3, v4
-; GFX11-NEXT: v_and_b32_e32 v8, -4, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v7, 3, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v9, v6
-; GFX11-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-NEXT: v_readfirstlane_b32 s2, v2
-; GFX11-NEXT: v_readfirstlane_b32 s3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
-; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX11-NEXT: s_cbranch_execnz .LBB18_1
-; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB18_3: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Loop Header: Depth=1
-; GFX11-NEXT: ; Child Loop BB18_4 Depth 2
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX11-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_mov_b32 s6, s5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-NEXT: v_add_f32_e32 v4, v4, v10
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v11, 0x400000, v4
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, v7, v4
-; GFX11-NEXT: v_and_or_b32 v5, v6, v9, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
-; GFX11-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
-; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-NEXT: v_readfirstlane_b32 s2, v2
-; GFX11-NEXT: v_readfirstlane_b32 s3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
-; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX11-NEXT: s_cbranch_execnz .LBB18_4
-; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
-; GFX11-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
-; GFX11-NEXT: v_mov_b32_e32 v6, v4
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB18_3
-; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v7, v4
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v9, v6
+; GFX11-TRUE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_1
+; GFX11-TRUE16-NEXT: ; %bb.2:
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v4
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB18_3: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: ; Child Loop BB18_4 Depth 2
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, v4, v10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
+; GFX11-TRUE16-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
+; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_4
+; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v4
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_3
+; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v9, v6
+; GFX11-FAKE16-NEXT: .LBB18_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_1
+; GFX11-FAKE16-NEXT: ; %bb.2:
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB18_3: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: ; Child Loop BB18_4 Depth 2
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v4, v4, v10
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
+; GFX11-FAKE16-NEXT: .LBB18_4: ; Parent Loop BB18_3 Depth=1
+; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_4
+; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB18_3 Depth=1
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v4
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_3
+; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x2
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index d70c365bf9e78..e72b47f55092c 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -4071,63 +4071,120 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; --------------------------------------------------------------------
define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_b32 s4, s16, -4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v4, s4
-; GFX12-NEXT: s_and_b32 s4, s16, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
-; GFX12-NEXT: s_not_b32 s6, s5
-; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB13_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB13_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX12-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v0, v0, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB13_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX12-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v0, v0, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB13_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -4172,57 +4229,108 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_addk_i32 s16, 0x200
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-NEXT: s_and_b32 s4, s16, -4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v4, s4
-; GFX11-NEXT: s_and_b32 s4, s16, 3
-; GFX11-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
-; GFX11-NEXT: s_not_b32 s6, s5
-; GFX11-NEXT: s_mov_b32 s5, 0
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB13_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_max_f32_e32 v0, v0, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX11-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB13_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v0, v0, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB13_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v0, v0, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB13_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -4479,86 +4587,142 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
}
define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_b32 s4, s16, -4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-NEXT: s_and_b32 s4, s16, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
-; GFX12-NEXT: s_not_b32 s6, s5
-; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, v4
-; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB14_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX12-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v0, v0, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB14_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_addk_i32 s16, 0x200
-; GFX942-NEXT: s_and_b32 s4, s16, -4
-; GFX942-NEXT: v_mov_b32_e32 v2, s4
-; GFX942-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen
-; GFX942-NEXT: s_and_b32 s4, s16, 3
-; GFX942-NEXT: s_lshl_b32 s6, s4, 3
-; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s6
-; GFX942-NEXT: s_not_b32 s7, s4
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX942-NEXT: s_movk_i32 s8, 0x7fff
-; GFX942-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX942-NEXT: buffer_wbl2 sc1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v0, v0, v3
-; GFX942-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX12-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v0, v0, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB14_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: s_addk_i32 s16, 0x200
+; GFX942-NEXT: s_and_b32 s4, s16, -4
+; GFX942-NEXT: v_mov_b32_e32 v2, s4
+; GFX942-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen
+; GFX942-NEXT: s_and_b32 s4, s16, 3
+; GFX942-NEXT: s_lshl_b32 s6, s4, 3
+; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX942-NEXT: s_not_b32 s7, s4
+; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX942-NEXT: s_movk_i32 s8, 0x7fff
+; GFX942-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX942-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX942-NEXT: v_add3_u32 v4, v4, v0, s8
; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX942-NEXT: s_nop 1
@@ -4578,56 +4742,106 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_addk_i32 s16, 0x200
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-NEXT: s_and_b32 s4, s16, -4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
-; GFX11-NEXT: s_and_b32 s4, s16, 3
-; GFX11-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
-; GFX11-NEXT: s_not_b32 s6, s5
-; GFX11-NEXT: s_mov_b32 s5, 0
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_max_f32_e32 v0, v0, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX11-NEXT: v_mov_b32_e32 v1, v4
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB14_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB14_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB14_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -4878,94 +5092,185 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
}
define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX12-NEXT: s_mov_b32 s4, exec_lo
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s5, s4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_and_b32_e32 v6, 3, v4
-; GFX12-NEXT: v_and_b32_e32 v8, -4, v4
-; GFX12-NEXT: v_lshlrev_b32_e32 v7, 3, v6
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
-; GFX12-NEXT: v_not_b32_e32 v9, v6
-; GFX12-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_readfirstlane_b32 s0, v0
-; GFX12-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-NEXT: v_readfirstlane_b32 s2, v2
-; GFX12-NEXT: v_readfirstlane_b32 s3, v3
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
-; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB15_1
-; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s4
-; GFX12-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB15_3: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Loop Header: Depth=1
-; GFX12-NEXT: ; Child Loop BB15_4 Depth 2
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX12-NEXT: s_mov_b32 s5, exec_lo
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s6, s5
-; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v4, v4, v10
-; GFX12-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v11, 0x400000, v4
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX12-NEXT: v_lshlrev_b32_e32 v4, v7, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v9, v4
-; GFX12-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
-; GFX12-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
-; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-NEXT: v_readfirstlane_b32 s0, v0
-; GFX12-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-NEXT: v_readfirstlane_b32 s2, v2
-; GFX12-NEXT: v_readfirstlane_b32 s3, v3
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX12-NEXT: s_cbranch_execnz .LBB15_4
-; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX12-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
-; GFX12-NEXT: v_mov_b32_e32 v6, v4
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB15_3
-; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v7, v4
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v9, v6
+; GFX12-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_1
+; GFX12-TRUE16-NEXT: ; %bb.2:
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v4
+; GFX12-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: ; Child Loop BB15_4 Depth 2
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v4, v4, v10
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
+; GFX12-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
+; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_4
+; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v4
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_3
+; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v9, v6
+; GFX12-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_1
+; GFX12-FAKE16-NEXT: ; %bb.2:
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-FAKE16-NEXT: .LBB15_3: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: ; Child Loop BB15_4 Depth 2
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v4, v4, v10
+; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
+; GFX12-FAKE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
+; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_4
+; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v4
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_3
+; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -5042,89 +5347,175 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amd
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v8, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_mov_b32 s6, s5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_b32_e32 v6, 3, v4
-; GFX11-NEXT: v_and_b32_e32 v8, -4, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v7, 3, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v9, v6
-; GFX11-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-NEXT: v_readfirstlane_b32 s2, v2
-; GFX11-NEXT: v_readfirstlane_b32 s3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
-; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX11-NEXT: s_cbranch_execnz .LBB15_1
-; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB15_3: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Loop Header: Depth=1
-; GFX11-NEXT: ; Child Loop BB15_4 Depth 2
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX11-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_mov_b32 s6, s5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-NEXT: v_max_f32_e32 v4, v4, v10
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v11, 0x400000, v4
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, v7, v4
-; GFX11-NEXT: v_and_or_b32 v5, v6, v9, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
-; GFX11-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
-; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-NEXT: v_readfirstlane_b32 s2, v2
-; GFX11-NEXT: v_readfirstlane_b32 s3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
-; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX11-NEXT: s_cbranch_execnz .LBB15_4
-; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX11-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
-; GFX11-NEXT: v_mov_b32_e32 v6, v4
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB15_3
-; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v7, v4
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v9, v6
+; GFX11-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_1
+; GFX11-TRUE16-NEXT: ; %bb.2:
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v4
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: ; Child Loop BB15_4 Depth 2
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v4, v4, v10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
+; GFX11-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
+; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_4
+; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v4
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_3
+; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v9, v6
+; GFX11-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_1
+; GFX11-FAKE16-NEXT: ; %bb.2:
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB15_3: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: ; Child Loop BB15_4 Depth 2
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v4, v4, v10
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
+; GFX11-FAKE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
+; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_4
+; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v4
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_3
+; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x2
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index 37e24bd18904e..7a77d8456b595 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -4071,63 +4071,120 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; --------------------------------------------------------------------
define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_b32 s4, s16, -4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v4, s4
-; GFX12-NEXT: s_and_b32 s4, s16, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
-; GFX12-NEXT: s_not_b32 s6, s5
-; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB13_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB13_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX12-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v0, v0, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB13_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX12-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v0, v0, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB13_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -4172,57 +4229,108 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_addk_i32 s16, 0x200
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-NEXT: s_and_b32 s4, s16, -4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v4, s4
-; GFX11-NEXT: s_and_b32 s4, s16, 3
-; GFX11-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
-; GFX11-NEXT: s_not_b32 s6, s5
-; GFX11-NEXT: s_mov_b32 s5, 0
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB13_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_min_f32_e32 v0, v0, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX11-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB13_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v0, v0, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB13_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v0, v0, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB13_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -4479,86 +4587,142 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
}
define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_b32 s4, s16, -4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-NEXT: s_and_b32 s4, s16, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
-; GFX12-NEXT: s_not_b32 s6, s5
-; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, v4
-; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB14_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX12-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v0, v0, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB14_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_addk_i32 s16, 0x200
-; GFX942-NEXT: s_and_b32 s4, s16, -4
-; GFX942-NEXT: v_mov_b32_e32 v2, s4
-; GFX942-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen
-; GFX942-NEXT: s_and_b32 s4, s16, 3
-; GFX942-NEXT: s_lshl_b32 s6, s4, 3
-; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s6
-; GFX942-NEXT: s_not_b32 s7, s4
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX942-NEXT: s_movk_i32 s8, 0x7fff
-; GFX942-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX942-NEXT: buffer_wbl2 sc1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_min_f32_e32 v0, v0, v3
-; GFX942-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX12-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v0, v0, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB14_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: s_addk_i32 s16, 0x200
+; GFX942-NEXT: s_and_b32 s4, s16, -4
+; GFX942-NEXT: v_mov_b32_e32 v2, s4
+; GFX942-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen
+; GFX942-NEXT: s_and_b32 s4, s16, 3
+; GFX942-NEXT: s_lshl_b32 s6, s4, 3
+; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX942-NEXT: s_not_b32 s7, s4
+; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX942-NEXT: s_movk_i32 s8, 0x7fff
+; GFX942-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX942-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX942-NEXT: v_add3_u32 v4, v4, v0, s8
; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX942-NEXT: s_nop 1
@@ -4578,56 +4742,106 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_addk_i32 s16, 0x200
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-NEXT: s_and_b32 s4, s16, -4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
-; GFX11-NEXT: s_and_b32 s4, s16, 3
-; GFX11-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
-; GFX11-NEXT: s_not_b32 s6, s5
-; GFX11-NEXT: s_mov_b32 s5, 0
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_min_f32_e32 v0, v0, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX11-NEXT: v_mov_b32_e32 v1, v4
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB14_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB14_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB14_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -4878,94 +5092,185 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
}
define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory(ptr addrspace(7) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX12-NEXT: s_mov_b32 s4, exec_lo
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s5, s4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_and_b32_e32 v6, 3, v4
-; GFX12-NEXT: v_and_b32_e32 v8, -4, v4
-; GFX12-NEXT: v_lshlrev_b32_e32 v7, 3, v6
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
-; GFX12-NEXT: v_not_b32_e32 v9, v6
-; GFX12-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_readfirstlane_b32 s0, v0
-; GFX12-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-NEXT: v_readfirstlane_b32 s2, v2
-; GFX12-NEXT: v_readfirstlane_b32 s3, v3
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
-; GFX12-NEXT: s_and_not1_wrexec_b32 s5, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB15_1
-; GFX12-NEXT: ; %bb.2:
-; GFX12-NEXT: s_mov_b32 exec_lo, s4
-; GFX12-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB15_3: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Loop Header: Depth=1
-; GFX12-NEXT: ; Child Loop BB15_4 Depth 2
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX12-NEXT: s_mov_b32 s5, exec_lo
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mov_b32 s6, s5
-; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v4, v4, v10
-; GFX12-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v11, 0x400000, v4
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX12-NEXT: v_lshlrev_b32_e32 v4, v7, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v9, v4
-; GFX12-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
-; GFX12-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
-; GFX12-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX12-NEXT: v_readfirstlane_b32 s0, v0
-; GFX12-NEXT: v_readfirstlane_b32 s1, v1
-; GFX12-NEXT: v_readfirstlane_b32 s2, v2
-; GFX12-NEXT: v_readfirstlane_b32 s3, v3
-; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX12-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX12-NEXT: s_cbranch_execnz .LBB15_4
-; GFX12-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX12-NEXT: s_mov_b32 exec_lo, s5
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
-; GFX12-NEXT: v_mov_b32_e32 v6, v4
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: s_cbranch_execnz .LBB15_3
-; GFX12-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v7, v4
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v9, v6
+; GFX12-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_1
+; GFX12-TRUE16-NEXT: ; %bb.2:
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v4
+; GFX12-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: ; Child Loop BB15_4 Depth 2
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v4, v4, v10
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
+; GFX12-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
+; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_4
+; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
+; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v4
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB15_3
+; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v9, v6
+; GFX12-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_1
+; GFX12-FAKE16-NEXT: ; %bb.2:
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX12-FAKE16-NEXT: .LBB15_3: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: ; Child Loop BB15_4 Depth 2
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v4, v4, v10
+; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
+; GFX12-FAKE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
+; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_4
+; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
+; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v4
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB15_3
+; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -5042,89 +5347,175 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amd
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v8, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
-; GFX11-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_mov_b32 s6, s5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_and_b32_e32 v6, 3, v4
-; GFX11-NEXT: v_and_b32_e32 v8, -4, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v7, 3, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v9, v6
-; GFX11-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-NEXT: v_readfirstlane_b32 s2, v2
-; GFX11-NEXT: v_readfirstlane_b32 s3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
-; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX11-NEXT: s_cbranch_execnz .LBB15_1
-; GFX11-NEXT: ; %bb.2:
-; GFX11-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-NEXT: v_lshlrev_b32_e32 v10, 16, v5
-; GFX11-NEXT: s_set_inst_prefetch_distance 0x1
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB15_3: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Loop Header: Depth=1
-; GFX11-NEXT: ; Child Loop BB15_4 Depth 2
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v4, v7, v6
-; GFX11-NEXT: s_mov_b32 s5, exec_lo
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_mov_b32 s6, s5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-NEXT: v_min_f32_e32 v4, v4, v10
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v11, 0x400000, v4
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, v7, v4
-; GFX11-NEXT: v_and_or_b32 v5, v6, v9, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
-; GFX11-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
-; GFX11-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: v_readfirstlane_b32 s1, v1
-; GFX11-NEXT: v_readfirstlane_b32 s2, v2
-; GFX11-NEXT: v_readfirstlane_b32 s3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
-; GFX11-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
-; GFX11-NEXT: s_and_not1_wrexec_b32 s6, s6
-; GFX11-NEXT: s_cbranch_execnz .LBB15_4
-; GFX11-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
-; GFX11-NEXT: s_mov_b32 exec_lo, s5
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
-; GFX11-NEXT: v_mov_b32_e32 v6, v4
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: s_cbranch_execnz .LBB15_3
-; GFX11-NEXT: ; %bb.6: ; %atomicrmw.end
-; GFX11-NEXT: s_set_inst_prefetch_distance 0x2
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v7, v4
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v9, v6
+; GFX11-TRUE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_1
+; GFX11-TRUE16-NEXT: ; %bb.2:
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v4
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB15_3: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: ; Child Loop BB15_4 Depth 2
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v4, v4, v10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
+; GFX11-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
+; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_4
+; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v4
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB15_3
+; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v9, v6
+; GFX11-FAKE16-NEXT: .LBB15_1: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_1
+; GFX11-FAKE16-NEXT: ; %bb.2:
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB15_3: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: ; Child Loop BB15_4 Depth 2
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v4, v4, v10
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v5, v11, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
+; GFX11-FAKE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
+; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_4
+; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB15_3 Depth=1
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v4
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB15_3
+; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x2
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll b/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll
index 85d1e02216ea1..e5ff94a4c81cb 100644
--- a/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll
@@ -616,33 +616,32 @@ define amdgpu_kernel void @v_fabs_fold_self_v2bf16(ptr addrspace(1) %out, ptr ad
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v2, v0, s[2:3]
+; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0x7fff, v2.h
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0x7fff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_mul_f32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0x7fff, v0.l
+; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0x7fff, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_dual_mul_f32 v0, v2, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_cndmask_b32 v1, v3, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: global_store_b32 v2, v1, s[0:1]
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX11-TRUE16-NEXT: global_store_b32 v2, v0, s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
;
; GFX11-FAKE16-LABEL: v_fabs_fold_self_v2bf16:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
index 187db078282d3..1c70bdf17fabd 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
@@ -11936,58 +11936,113 @@ define void @flat_system_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; --------------------------------------------------------------------
define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB46_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB46_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB46_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB46_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -12032,54 +12087,105 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB46_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB46_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB46_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB46_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -12279,86 +12385,143 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(pt
}
define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB47_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB47_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX942-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_mov_b64 s[0:1], 0x7fe
-; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
-; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
-; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: flat_load_dword v3, v[0:1]
-; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
-; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
-; GFX942-NEXT: v_not_b32_e32 v5, v5
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX942-NEXT: s_movk_i32 s2, 0x7fff
-; GFX942-NEXT: .LBB47_1: ; %atomicrmw.start
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_add_f32_e32 v2, v2, v6
-; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB47_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB47_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: s_mov_b64 s[0:1], 0x7fe
+; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
+; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v5
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX942-NEXT: s_mov_b32 s0, 0xffff
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
+; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX942-NEXT: s_movk_i32 s2, 0x7fff
+; GFX942-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX942-NEXT: v_add3_u32 v7, v7, v2, s2
; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX942-NEXT: s_nop 1
@@ -12380,56 +12543,108 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB47_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB47_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB47_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB47_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -12635,61 +12850,118 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
}
define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB48_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB48_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB48_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB48_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -12737,56 +13009,108 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB48_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB48_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB48_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB48_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -14332,56 +14656,109 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no
}
define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB53_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: v_add_f32_e32 v3, v3, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB53_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v3
+; GFX12-TRUE16-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB53_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB53_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -14425,52 +14802,101 @@ define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB53_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_add_f32_e32 v3, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB53_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB53_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB53_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -14665,62 +15091,120 @@ define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(pt
}
define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB54_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB54_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB54_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -14767,56 +15251,108 @@ define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB54_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB54_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB54_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
index a37481d1ae20d..383d1aabad992 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
@@ -9707,58 +9707,113 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; --------------------------------------------------------------------
define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB36_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -9803,54 +9858,105 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB36_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -10051,61 +10157,118 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(pt
}
define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB37_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10152,56 +10315,108 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB37_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -10408,61 +10623,118 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
}
define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB38_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10510,56 +10782,108 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB38_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -10766,56 +11090,109 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
}
define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: v_max_num_f32_e32 v3, v3, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB39_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v3
+; GFX12-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10859,52 +11236,101 @@ define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_max_f32_e32 v3, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB39_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -12444,62 +12870,120 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no
}
define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB44_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -12546,56 +13030,108 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB44_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
index 63fa32e40a591..85bf71b54bae4 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
@@ -9707,58 +9707,113 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; --------------------------------------------------------------------
define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB36_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -9803,54 +9858,105 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB36_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -10051,61 +10157,118 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(pt
}
define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB37_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10152,56 +10315,108 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB37_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -10408,61 +10623,118 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
}
define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB38_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10510,56 +10782,108 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB38_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -10766,56 +11090,109 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
}
define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: v_min_num_f32_e32 v3, v3, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB39_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v3
+; GFX12-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10859,52 +11236,101 @@ define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_min_f32_e32 v3, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB39_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -12444,62 +12870,120 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no
}
define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB44_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -12546,56 +13030,108 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB44_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
index 6a90498470a25..cad3a2a30a55e 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
@@ -9378,58 +9378,113 @@ define void @flat_system_atomic_fsub_noret_f16__offset12b_pos(ptr %ptr, half %va
; --------------------------------------------------------------------
define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fsub_ret_bf16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB32_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB32_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB32_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB32_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fsub_ret_bf16:
; GFX942: ; %bb.0:
@@ -9474,54 +9529,105 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fsub_ret_bf16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB32_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB32_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB32_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB32_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fsub_ret_bf16:
; GFX10: ; %bb.0:
@@ -9721,61 +9827,118 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 {
}
define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB33_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB33_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB33_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB33_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
; GFX942: ; %bb.0:
@@ -9822,56 +9985,108 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB33_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB33_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB33_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB33_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
; GFX10: ; %bb.0:
@@ -10077,61 +10292,118 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
}
define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB34_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB34_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB34_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
; GFX942: ; %bb.0:
@@ -10179,56 +10451,108 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB34_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB34_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB34_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
; GFX10: ; %bb.0:
@@ -10434,56 +10758,109 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
}
define void @flat_agent_atomic_fsub_noret_bf16(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fsub_noret_bf16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB35_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: v_sub_f32_e32 v3, v3, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB35_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fsub_noret_bf16:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v3
+; GFX12-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB35_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fsub_noret_bf16:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB35_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fsub_noret_bf16:
; GFX942: ; %bb.0:
@@ -10527,52 +10904,101 @@ define void @flat_agent_atomic_fsub_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fsub_noret_bf16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB35_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_sub_f32_e32 v3, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB35_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_noret_bf16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB35_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_noret_bf16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB35_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fsub_noret_bf16:
; GFX10: ; %bb.0:
@@ -12107,62 +12533,120 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr %ptr,
}
define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB40_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB40_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB40_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB40_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
; GFX942: ; %bb.0:
@@ -12209,56 +12693,108 @@ define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB40_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB40_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB40_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB40_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-f16-hw.ll b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-f16-hw.ll
index cc7f9d7a7cf3f..5eb6c01a82695 100644
--- a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-f16-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-f16-hw.ll
@@ -1082,15 +1082,12 @@ define <4 x i8> @to_fp8_v4f16(<4 x half> %x) {
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v2.h, v1
-; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v2.l, v0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX1250-TRUE16-NEXT: v_dual_lshlrev_b32 v0, 16, v1 :: v_dual_lshrrev_b32 v1, 8, v2
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v2.l, v1
+; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v0.l, v0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX1250-TRUE16-NEXT: v_dual_lshrrev_b32 v1, 8, v0 :: v_dual_lshrrev_b32 v3, 24, v3
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1250-FAKE16-LABEL: to_fp8_v4f16:
@@ -1350,15 +1347,12 @@ define <4 x i8> @to_bf8_v4f16(<4 x half> %x) {
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf8_f16_e64 v2.h, v1
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf8_f16_e64 v2.l, v0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX1250-TRUE16-NEXT: v_dual_lshlrev_b32 v0, 16, v1 :: v_dual_lshrrev_b32 v1, 8, v2
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX1250-TRUE16-NEXT: v_cvt_pk_bf8_f16_e64 v2.l, v1
+; GFX1250-TRUE16-NEXT: v_cvt_pk_bf8_f16_e64 v0.l, v0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX1250-TRUE16-NEXT: v_dual_lshrrev_b32 v1, 8, v0 :: v_dual_lshrrev_b32 v3, 24, v3
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1250-FAKE16-LABEL: to_bf8_v4f16:
diff --git a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll
index 4995e777939c6..ded6a072d8166 100644
--- a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll
@@ -1606,88 +1606,90 @@ define i8 @to_fp8_bf16(bfloat %x) {
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX1200-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX1200-NEXT: v_lshrrev_b16 v0.l, 8, v0.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX1200-NEXT: v_and_b16 v0.l, v0.l, 0x80
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_frexp_mant_f32_e32 v2, v5
; GFX1200-NEXT: v_frexp_exp_i32_f32_e32 v6, v5
-; GFX1200-NEXT: v_and_b16 v0.l, v0.l, 0x80
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1200-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX1200-NEXT: v_xor_b16 v1.l, v6.l, -1
; GFX1200-NEXT: v_or_b32_e32 v4, 0x400000, v2
; GFX1200-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1200-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_min_u16 v1.l, v1.l, 15
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_cndmask_b32_e32 v7, v3, v4, vcc_lo
-; GFX1200-NEXT: v_sub_nc_u16 v1.h, v1.l, 1 clamp
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-NEXT: v_sub_nc_u16 v1.h, v1.l, 1 clamp
; GFX1200-NEXT: v_and_b16 v2.l, v7.h, 0x7f
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX1200-NEXT: v_lshlrev_b16 v2.h, v1.h, 1
; GFX1200-NEXT: v_and_b16 v3.l, v7.h, 7
; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 19, v7
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1200-NEXT: v_or_b16 v3.h, v2.l, 0x80
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX1200-NEXT: v_add_nc_u16 v2.h, v2.h, -1
; GFX1200-NEXT: v_lshrrev_b16 v2.l, 4, v2.l
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1200-NEXT: v_lshrrev_b16 v1.h, v1.h, v3.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1200-NEXT: v_and_b16 v2.h, v3.h, v2.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
; GFX1200-NEXT: v_and_b16 v3.l, v2.l, 1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
; GFX1200-NEXT: v_lshrrev_b16 v4.h, v1.l, v3.h
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1200-NEXT: v_or_b16 v2.h, v4.l, v3.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
; GFX1200-NEXT: v_and_b16 v3.l, v4.h, 1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v1.l
; GFX1200-NEXT: v_and_b16 v2.h, v7.l, v2.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_mov_b16_e32 v4.l, v8.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_add_nc_u16 v2.l, v2.l, v2.h
-; GFX1200-NEXT: v_or_b16 v3.l, v4.l, v3.l
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-NEXT: v_or_b16 v3.l, v4.l, v3.l
; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v2.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_and_b16 v1.l, v1.h, v3.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1200-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, 0, s0
; GFX1200-NEXT: v_cmp_eq_f32_e64 s0, 0, v5
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b16 v1.l, 0, v1.l, vcc_lo
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_add_nc_u16 v0.h, v4.h, v1.l
; GFX1200-NEXT: v_add_nc_u16 v1.l, v6.l, v3.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v0.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_add_nc_u16 v1.l, v1.l, 6
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b16 v1.h, 0, 8, vcc_lo
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX1200-NEXT: v_lshlrev_b16 v2.h, 3, v1.l
; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, 0, vcc_lo
; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v1.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1200-NEXT: v_or_b16 v1.h, v0.l, v1.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_or_b16 v2.h, v0.l, v2.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_or_b16 v0.h, v1.h, v0.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1200-NEXT: v_or_b16 v1.l, v2.h, v2.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_cndmask_b16 v0.h, v1.l, v0.h, vcc_lo
; GFX1200-NEXT: v_cmp_o_f32_e32 vcc_lo, v5, v5
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1200-NEXT: v_cndmask_b16 v0.l, v0.h, v0.l, s0
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1200-NEXT: v_cndmask_b16 v0.l, 0x7f, v0.l, vcc_lo
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
@@ -1695,73 +1697,74 @@ define i8 @to_fp8_bf16(bfloat %x) {
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0x80
; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v0.l, 8, v0.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX1250-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 0x80
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1250-TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v7, v6
; GFX1250-TRUE16-NEXT: v_frexp_mant_f32_e32 v2, v6
-; GFX1250-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 0x80
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1250-TRUE16-NEXT: v_xor_b16 v1.l, v7.l, -1
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v2, s0
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v2, s0
; GFX1250-TRUE16-NEXT: v_min_u16 v1.l, v1.l, 15
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1250-TRUE16-NEXT: v_and_b16 v2.h, v2.l, 0x7f
; GFX1250-TRUE16-NEXT: v_and_b16 v3.l, v2.l, 7
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1250-TRUE16-NEXT: v_sub_nc_u16 v1.h, v1.l, 1 clamp
-; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v4.h, 4, v2.h
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v4.h, 4, v2.h
; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.h, v1.h, 1
; GFX1250-TRUE16-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX1250-TRUE16-NEXT: v_add_nc_u16 v3.l, v3.h, -1
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.h, v2.l, v4.l, 0x7f bitop3:0xec
; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v2.l, 3, v2.l
; GFX1250-TRUE16-NEXT: v_and_b16 v4.l, v4.h, 1
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.h, v2.h, v3.l, 0x80 bitop3:0xc8
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v3.l, v1.l, v3.h
; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v1.h, v1.h, v3.h
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.l, v2.l, v5.l, v4.l bitop3:0xe0
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.l, v2.l, v5.l, v4.l bitop3:0xe0
; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2.h
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1250-TRUE16-NEXT: v_and_b16 v2.h, v3.l, 1
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1250-TRUE16-NEXT: v_add_nc_u16 v2.l, v4.h, v2.l
; GFX1250-TRUE16-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v1.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e64 s0, 7, v2.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-TRUE16-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v1.h, v4.l, v2.h bitop3:0xe0
; GFX1250-TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, 0, s0
; GFX1250-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, v1.l, vcc_lo
-; GFX1250-TRUE16-NEXT: v_add_nc_u16 v0.h, v7.l, v4.l
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: v_add_nc_u16 v0.h, v7.l, v4.l
; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.l, v3.l, v1.l
-; GFX1250-TRUE16-NEXT: v_add_nc_u16 v0.h, v0.h, 6
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: v_add_nc_u16 v0.h, v0.h, 6
; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v1.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.l, 3, v0.h
; GFX1250-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, 0, vcc_lo
; GFX1250-TRUE16-NEXT: v_cndmask_b16 v2.h, 0, 8, vcc_lo
; GFX1250-TRUE16-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v0.h
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v0.l, v1.h, v2.l bitop3:0xfe
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v0.l, v1.l, v2.h bitop3:0xfe
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v1.l, vcc_lo
; GFX1250-TRUE16-NEXT: v_cmp_o_f32_e32 vcc_lo, v6, v6
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, v0.l, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x7f, v0.l, vcc_lo
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
@@ -2632,178 +2635,183 @@ define <4 x i8> @to_e5m3fnu_v4f32(<4 x float> %x) {
; GFX1200-NEXT: s_wait_kmcnt 0x0
; GFX1200-NEXT: v_frexp_exp_i32_f32_e32 v4, v1
; GFX1200-NEXT: v_frexp_mant_f32_e32 v5, v1
-; GFX1200-NEXT: v_frexp_exp_i32_f32_e32 v6, v0
+; GFX1200-NEXT: v_frexp_exp_i32_f32_e32 v7, v0
; GFX1200-NEXT: s_mov_b32 s1, 0x7fffff
-; GFX1200-NEXT: v_frexp_mant_f32_e32 v7, v0
+; GFX1200-NEXT: v_frexp_mant_f32_e32 v6, v0
; GFX1200-NEXT: v_sub_nc_u32_e32 v8, 7, v4
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: v_and_or_b32 v13, v5, s1, 0x800000
-; GFX1200-NEXT: v_sub_nc_u32_e32 v11, 7, v6
-; GFX1200-NEXT: v_and_b32_e32 v14, 0x7ffff, v5
-; GFX1200-NEXT: v_and_or_b32 v17, v7, s1, 0x800000
+; GFX1200-NEXT: v_and_or_b32 v12, v5, s1, 0x800000
+; GFX1200-NEXT: v_and_b32_e32 v13, 0x7ffff, v5
+; GFX1200-NEXT: v_sub_nc_u32_e32 v15, 7, v7
+; GFX1200-NEXT: v_and_or_b32 v17, v6, s1, 0x800000
; GFX1200-NEXT: v_min_u32_e32 v8, 31, v8
-; GFX1200-NEXT: v_bfe_u32 v15, v5, 20, 3
-; GFX1200-NEXT: v_min_u32_e32 v11, 31, v11
-; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v14
+; GFX1200-NEXT: v_bfe_u32 v14, v5, 20, 3
+; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v13
+; GFX1200-NEXT: v_min_u32_e32 v15, 31, v15
; GFX1200-NEXT: v_lshrrev_b32_e32 v5, 19, v5
; GFX1200-NEXT: v_sub_nc_u32_e64 v16, v8, 1 clamp
-; GFX1200-NEXT: v_lshrrev_b32_e32 v22, v8, v13
-; GFX1200-NEXT: v_sub_nc_u32_e64 v18, v11, 1 clamp
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc_lo
-; GFX1200-NEXT: v_lshrrev_b32_e32 v23, v11, v17
-; GFX1200-NEXT: v_bfe_u32 v20, v13, 0, v16
-; GFX1200-NEXT: v_lshrrev_b32_e32 v13, v16, v13
-; GFX1200-NEXT: v_bfe_u32 v21, v17, 0, v18
-; GFX1200-NEXT: v_and_or_b32 v14, v15, 1, v14
-; GFX1200-NEXT: v_lshrrev_b32_e32 v17, v18, v17
-; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v20
+; GFX1200-NEXT: v_lshrrev_b32_e32 v22, v8, v12
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc_lo
+; GFX1200-NEXT: v_sub_nc_u32_e64 v21, v15, 1 clamp
+; GFX1200-NEXT: v_cmp_ne_u32_e64 s0, 0, v15
+; GFX1200-NEXT: v_bfe_u32 v20, v12, 0, v16
+; GFX1200-NEXT: v_lshrrev_b32_e32 v12, v16, v12
+; GFX1200-NEXT: v_and_or_b32 v13, v14, 1, v13
+; GFX1200-NEXT: v_bfe_u32 v24, v17, 0, v21
; GFX1200-NEXT: v_frexp_exp_i32_f32_e32 v10, v3
-; GFX1200-NEXT: v_frexp_mant_f32_e32 v9, v3
-; GFX1200-NEXT: v_and_b32_e32 v5, v5, v14
-; GFX1200-NEXT: v_frexp_exp_i32_f32_e32 v12, v2
+; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v20
+; GFX1200-NEXT: v_and_b32_e32 v18, 0x7ffff, v6
+; GFX1200-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX1200-NEXT: v_lshrrev_b32_e32 v13, v21, v17
+; GFX1200-NEXT: v_sub_nc_u32_e32 v23, 7, v10
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v20, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v21
-; GFX1200-NEXT: v_sub_nc_u32_e32 v14, 7, v10
-; GFX1200-NEXT: v_add_nc_u32_e32 v5, v15, v5
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v24
+; GFX1200-NEXT: v_add_nc_u32_e32 v5, v14, v5
+; GFX1200-NEXT: v_frexp_mant_f32_e32 v9, v3
+; GFX1200-NEXT: v_min_u32_e32 v23, 31, v23
; GFX1200-NEXT: v_and_or_b32 v16, v22, 1, v20
+; GFX1200-NEXT: v_lshrrev_b32_e32 v20, v15, v17
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v21, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cndmask_b32_e64 v24, 0, 1, vcc_lo
; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
-; GFX1200-NEXT: v_min_u32_e32 v14, 31, v14
-; GFX1200-NEXT: v_and_or_b32 v20, v9, s1, 0x800000
-; GFX1200-NEXT: v_and_b32_e32 v13, v13, v16
-; GFX1200-NEXT: v_and_b32_e32 v19, 0x7ffff, v7
-; GFX1200-NEXT: v_and_or_b32 v18, v23, 1, v21
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cndmask_b32_e32 v8, 0, v13, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v11
-; GFX1200-NEXT: v_and_b32_e32 v16, v17, v18
-; GFX1200-NEXT: v_sub_nc_u32_e32 v13, 7, v12
-; GFX1200-NEXT: v_lshrrev_b32_e32 v18, v14, v20
-; GFX1200-NEXT: v_add_nc_u32_e32 v8, v22, v8
-; GFX1200-NEXT: v_bfe_u32 v22, v9, 20, 3
+; GFX1200-NEXT: v_sub_nc_u32_e64 v14, v23, 1 clamp
+; GFX1200-NEXT: v_and_b32_e32 v12, v12, v16
+; GFX1200-NEXT: v_frexp_exp_i32_f32_e32 v11, v2
+; GFX1200-NEXT: v_and_or_b32 v16, v20, 1, v24
+; GFX1200-NEXT: v_bfe_u32 v19, v6, 20, 3
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e32 v11, 0, v16, vcc_lo
+; GFX1200-NEXT: v_dual_cndmask_b32 v8, 0, v12 :: v_dual_and_b32 v21, 0x7ffff, v9
; GFX1200-NEXT: v_cmp_lt_i32_e32 vcc_lo, 7, v5
-; GFX1200-NEXT: v_sub_nc_u32_e64 v16, v14, 1 clamp
-; GFX1200-NEXT: v_cmp_lt_i32_e64 s0, 7, v8
-; GFX1200-NEXT: v_min_u32_e32 v13, 31, v13
+; GFX1200-NEXT: v_and_b32_e32 v13, v13, v16
+; GFX1200-NEXT: v_and_or_b32 v12, v9, s1, 0x800000
+; GFX1200-NEXT: v_lshrrev_b32_e32 v6, 19, v6
+; GFX1200-NEXT: v_add_nc_u32_e32 v8, v22, v8
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v5, v5, 0, vcc_lo
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, v13, s0
; GFX1200-NEXT: v_add_co_ci_u32_e64 v4, null, 14, v4, vcc_lo
+; GFX1200-NEXT: v_cmp_lt_i32_e64 s0, 7, v8
+; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v18
+; GFX1200-NEXT: v_frexp_mant_f32_e32 v18, v2
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1200-NEXT: v_lshl_or_b32 v5, v4, 3, v5
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v8, v8, 0, s0
; GFX1200-NEXT: v_cndmask_b32_e64 v15, 0, 8, s0
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_lshl_or_b32 v5, v4, 3, v5
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc_lo
+; GFX1200-NEXT: v_and_b32_e32 v22, 0x7ffff, v18
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX1200-NEXT: v_or_b32_e32 v8, v15, v8
-; GFX1200-NEXT: v_add_nc_u32_e32 v11, v23, v11
-; GFX1200-NEXT: v_bfe_u32 v15, v20, 0, v16
-; GFX1200-NEXT: v_lshrrev_b32_e32 v16, v16, v20
-; GFX1200-NEXT: v_sub_nc_u32_e64 v20, v13, 1 clamp
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_cmp_lt_i32_e32 vcc_lo, 7, v11
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v11, v11, 0, vcc_lo
+; GFX1200-NEXT: v_add_nc_u32_e32 v13, v20, v13
+; GFX1200-NEXT: v_bfe_u32 v15, v12, 0, v14
+; GFX1200-NEXT: v_and_or_b32 v16, v19, 1, v16
+; GFX1200-NEXT: v_cmp_lt_i32_e32 vcc_lo, 7, v13
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1200-NEXT: v_and_b32_e32 v6, v6, v16
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, v13, 0, vcc_lo
; GFX1200-NEXT: v_cndmask_b32_e64 v17, 0, 8, vcc_lo
; GFX1200-NEXT: v_cmp_gt_i32_e32 vcc_lo, 1, v4
-; GFX1200-NEXT: v_or_b32_e32 v11, v17, v11
+; GFX1200-NEXT: v_add_nc_u32_e32 v6, v19, v6
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1200-NEXT: v_or_b32_e32 v13, v17, v13
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e32 v4, v5, v8, vcc_lo
+; GFX1200-NEXT: v_sub_nc_u32_e32 v8, 7, v11
; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v15
-; GFX1200-NEXT: v_frexp_mant_f32_e32 v15, v2
-; GFX1200-NEXT: v_and_b32_e32 v21, 0x7ffff, v9
-; GFX1200-NEXT: v_bfe_u32 v17, v7, 20, 3
-; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 19, v7
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v19
-; GFX1200-NEXT: v_and_or_b32 v19, v15, s1, 0x800000
-; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 19, v9
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1200-NEXT: v_and_or_b32 v5, v18, 1, v5
+; GFX1200-NEXT: v_lshrrev_b32_e32 v5, v23, v12
+; GFX1200-NEXT: v_lshrrev_b32_e32 v12, v14, v12
+; GFX1200-NEXT: v_and_or_b32 v14, v18, s1, 0x800000
+; GFX1200-NEXT: v_min_u32_e32 v8, 31, v8
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc_lo
; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v21
-; GFX1200-NEXT: v_lshrrev_b32_e32 v23, v13, v19
-; GFX1200-NEXT: v_and_b32_e32 v5, v16, v5
-; GFX1200-NEXT: v_bfe_u32 v16, v19, 0, v20
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v21, 0, 1, vcc_lo
-; GFX1200-NEXT: v_and_or_b32 v8, v17, 1, v8
-; GFX1200-NEXT: v_lshrrev_b32_e32 v19, v20, v19
-; GFX1200-NEXT: v_bfe_u32 v20, v15, 20, 3
-; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v16
-; GFX1200-NEXT: v_and_or_b32 v21, v22, 1, v21
-; GFX1200-NEXT: v_and_b32_e32 v7, v7, v8
+; GFX1200-NEXT: v_bfe_u32 v17, v9, 20, 3
+; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 19, v9
+; GFX1200-NEXT: v_sub_nc_u32_e64 v20, v8, 1 clamp
+; GFX1200-NEXT: v_and_or_b32 v15, v5, 1, v15
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v14
-; GFX1200-NEXT: v_and_b32_e32 v14, 0x7ffff, v15
-; GFX1200-NEXT: v_and_b32_e32 v9, v9, v21
-; GFX1200-NEXT: v_lshrrev_b32_e32 v15, 19, v15
-; GFX1200-NEXT: v_and_or_b32 v16, v23, 1, v16
+; GFX1200-NEXT: v_lshrrev_b32_e32 v21, v8, v14
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_and_b32_e32 v12, v12, v15
+; GFX1200-NEXT: v_bfe_u32 v15, v14, 0, v20
+; GFX1200-NEXT: v_and_or_b32 v16, v17, 1, v16
+; GFX1200-NEXT: v_lshrrev_b32_e32 v14, v20, v14
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v15
+; GFX1200-NEXT: v_and_b32_e32 v9, v9, v16
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v14
-; GFX1200-NEXT: v_add_nc_u32_e32 v9, v22, v9
-; GFX1200-NEXT: v_add_nc_u32_e32 v7, v17, v7
-; GFX1200-NEXT: v_and_b32_e32 v8, v19, v16
-; GFX1200-NEXT: v_add_nc_u32_e32 v5, v18, v5
+; GFX1200-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v23
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_add_nc_u32_e32 v9, v17, v9
+; GFX1200-NEXT: v_and_or_b32 v15, v21, 1, v15
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v13
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_and_or_b32 v14, v20, 1, v14
+; GFX1200-NEXT: v_cndmask_b32_e32 v12, 0, v12, vcc_lo
+; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v22
+; GFX1200-NEXT: v_bfe_u32 v22, v18, 20, 3
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1200-NEXT: v_add_nc_u32_e32 v5, v5, v12
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e32 v8, 0, v8, vcc_lo
+; GFX1200-NEXT: v_cndmask_b32_e64 v20, 0, 1, vcc_lo
+; GFX1200-NEXT: v_and_b32_e32 v12, v14, v15
+; GFX1200-NEXT: v_lshrrev_b32_e32 v15, 19, v18
+; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_and_or_b32 v14, v22, 1, v20
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e32 v8, 0, v12, vcc_lo
; GFX1200-NEXT: v_cmp_lt_i32_e32 vcc_lo, 7, v5
-; GFX1200-NEXT: v_and_b32_e32 v13, v15, v14
+; GFX1200-NEXT: v_and_b32_e32 v12, v15, v14
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1200-NEXT: v_add_nc_u32_e32 v8, v23, v8
+; GFX1200-NEXT: v_add_nc_u32_e32 v8, v21, v8
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v5, v5, 0, vcc_lo
; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 8, vcc_lo
; GFX1200-NEXT: v_cmp_lt_i32_e32 vcc_lo, 7, v9
-; GFX1200-NEXT: v_add_nc_u32_e32 v13, v20, v13
+; GFX1200-NEXT: v_add_nc_u32_e32 v12, v22, v12
; GFX1200-NEXT: v_cmp_lt_i32_e64 s0, 7, v8
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1200-NEXT: v_or_b32_e32 v5, v14, v5
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v9, v9, 0, vcc_lo
; GFX1200-NEXT: v_add_co_ci_u32_e64 v10, null, 14, v10, vcc_lo
-; GFX1200-NEXT: v_cmp_lt_i32_e32 vcc_lo, 7, v13
+; GFX1200-NEXT: v_cmp_lt_i32_e32 vcc_lo, 7, v12
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v8, v8, 0, s0
; GFX1200-NEXT: v_cndmask_b32_e64 v15, 0, 8, s0
; GFX1200-NEXT: v_lshl_or_b32 v9, v10, 3, v9
-; GFX1200-NEXT: v_cmp_gt_i32_e64 s0, 1, v10
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v13, v13, 0, vcc_lo
-; GFX1200-NEXT: v_add_co_ci_u32_e64 v12, null, 14, v12, vcc_lo
-; GFX1200-NEXT: v_cmp_lt_i32_e32 vcc_lo, 7, v7
+; GFX1200-NEXT: v_cndmask_b32_e64 v12, v12, 0, vcc_lo
+; GFX1200-NEXT: v_add_co_ci_u32_e64 v11, null, 14, v11, vcc_lo
+; GFX1200-NEXT: v_cmp_lt_i32_e32 vcc_lo, 7, v6
; GFX1200-NEXT: v_or_b32_e32 v8, v15, v8
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v5, v9, v5, s0
-; GFX1200-NEXT: v_lshl_or_b32 v13, v12, 3, v13
-; GFX1200-NEXT: v_cmp_gt_i32_e64 s0, 1, v12
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX1200-NEXT: v_lshl_or_b32 v12, v11, 3, v12
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v7, v7, 0, vcc_lo
-; GFX1200-NEXT: v_add_co_ci_u32_e64 v6, null, 14, v6, vcc_lo
+; GFX1200-NEXT: v_cndmask_b32_e64 v6, v6, 0, vcc_lo
+; GFX1200-NEXT: v_add_co_ci_u32_e64 v7, null, 14, v7, vcc_lo
+; GFX1200-NEXT: v_cmp_gt_i32_e32 vcc_lo, 1, v10
+; GFX1200-NEXT: v_lshl_or_b32 v6, v7, 3, v6
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e32 v5, v9, v5, vcc_lo
+; GFX1200-NEXT: v_cmp_gt_i32_e32 vcc_lo, 1, v11
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e32 v8, v12, v8, vcc_lo
; GFX1200-NEXT: v_cmp_neq_f32_e32 vcc_lo, 0, v3
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v8, v13, v8, s0
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1200-NEXT: v_lshl_or_b32 v7, v6, 3, v7
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc_lo
; GFX1200-NEXT: v_cmp_neq_f32_e32 vcc_lo, 0, v2
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e32 v8, 0, v8, vcc_lo
+; GFX1200-NEXT: v_cmp_gt_i32_e32 vcc_lo, 1, v7
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e32 v6, v6, v13, vcc_lo
; GFX1200-NEXT: v_cmp_o_f32_e32 vcc_lo, v3, v3
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e32 v3, 0xff, v5, vcc_lo
@@ -2812,35 +2820,30 @@ define <4 x i8> @to_e5m3fnu_v4f32(<4 x float> %x) {
; GFX1200-NEXT: v_lshlrev_b16 v2.l, 8, v3.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e32 v5, 0xff, v8, vcc_lo
-; GFX1200-NEXT: v_cmp_gt_i32_e32 vcc_lo, 1, v6
-; GFX1200-NEXT: v_and_b16 v2.h, v5.l, 0xff
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e32 v6, v7, v11, vcc_lo
; GFX1200-NEXT: v_cmp_neq_f32_e32 vcc_lo, 0, v1
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1200-NEXT: v_or_b16 v2.h, v2.h, v2.l
+; GFX1200-NEXT: v_and_b16 v2.h, v5.l, 0xff
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_or_b16 v2.l, v2.h, v2.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc_lo
; GFX1200-NEXT: v_cmp_neq_f32_e32 vcc_lo, 0, v0
+; GFX1200-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e32 v3, 0, v6, vcc_lo
+; GFX1200-NEXT: v_cndmask_b32_e32 v6, 0, v6, vcc_lo
; GFX1200-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v1
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e32 v1, 0xff, v4, vcc_lo
; GFX1200-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v0
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_lshlrev_b16 v2.l, 8, v1.l
+; GFX1200-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e32 v0, 0xff, v3, vcc_lo
-; GFX1200-NEXT: v_mov_b16_e32 v3.l, v2.h
-; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_cndmask_b32_e32 v0, 0xff, v6, vcc_lo
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-NEXT: v_and_b16 v0.l, v0.l, 0xff
-; GFX1200-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_or_b16 v0.l, v0.l, v2.l
-; GFX1200-NEXT: v_lshrrev_b32_e32 v3, 24, v3
-; GFX1200-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX1200-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-TRUE16-LABEL: to_e5m3fnu_v4f32:
diff --git a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-widen.ll b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-widen.ll
index ddc18105b30ed..7542c8dad863f 100644
--- a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-widen.ll
+++ b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-widen.ll
@@ -610,404 +610,392 @@ define <5 x i8> @to_fp8_v5f16(<5 x half> %x) {
; GFX1200-NEXT: s_wait_kmcnt 0x0
; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v3.l, v2.l
; GFX1200-NEXT: v_frexp_mant_f16_e32 v3.h, v2.l
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v4.h, v2.h
; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v5.h, v2.h
-; GFX1200-NEXT: v_lshrrev_b16 v4.l, 8, v2.l
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v4.h, v2.h
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v6.l, v1.l
; GFX1200-NEXT: v_sub_nc_u16 v5.l, 2, v3.l
; GFX1200-NEXT: v_and_b16 v6.h, v3.h, 0x3ff
; GFX1200-NEXT: v_and_b16 v7.l, v3.h, 63
; GFX1200-NEXT: v_sub_nc_u16 v8.l, 2, v5.h
-; GFX1200-NEXT: v_and_b16 v9.l, v4.h, 63
-; GFX1200-NEXT: v_min_u16 v5.l, v5.l, 15
-; GFX1200-NEXT: v_or_b16 v10.l, v6.h, 0x400
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, 7, v6.h
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.l
; GFX1200-NEXT: v_lshrrev_b16 v3.h, 6, v3.h
+; GFX1200-NEXT: v_min_u16 v5.l, v5.l, 15
+; GFX1200-NEXT: v_lshrrev_b16 v10.l, 7, v6.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.l
+; GFX1200-NEXT: v_or_b16 v6.h, v6.h, 0x400
+; GFX1200-NEXT: v_min_u16 v7.l, v8.l, 15
; GFX1200-NEXT: v_sub_nc_u16 v8.h, v5.l, 1 clamp
-; GFX1200-NEXT: v_min_u16 v8.l, v8.l, 15
-; GFX1200-NEXT: v_and_b16 v11.l, v6.h, 1
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v9.l
-; GFX1200-NEXT: v_lshlrev_b16 v10.h, v8.h, 1
-; GFX1200-NEXT: v_sub_nc_u16 v9.l, v8.l, 1 clamp
; GFX1200-NEXT: v_and_b16 v7.h, v4.h, 0x3ff
-; GFX1200-NEXT: v_or_b16 v11.l, v12.l, v11.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc_lo
-; GFX1200-NEXT: v_add_nc_u16 v10.h, v10.h, -1
-; GFX1200-NEXT: v_lshrrev_b16 v11.h, v5.l, v10.l
-; GFX1200-NEXT: v_or_b16 v7.l, v7.h, 0x400
-; GFX1200-NEXT: v_and_b16 v3.h, v3.h, v11.l
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0
+; GFX1200-NEXT: v_and_b16 v9.l, v4.h, 63
+; GFX1200-NEXT: v_sub_nc_u16 v12.l, v7.l, 1 clamp
+; GFX1200-NEXT: v_lshlrev_b16 v10.h, v8.h, 1
+; GFX1200-NEXT: v_or_b16 v11.h, v7.h, 0x400
+; GFX1200-NEXT: v_sub_nc_u16 v9.h, 2, v6.l
; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v7.h
-; GFX1200-NEXT: v_and_b16 v10.h, v10.l, v10.h
-; GFX1200-NEXT: v_lshrrev_b16 v8.h, v8.h, v10.l
+; GFX1200-NEXT: v_lshlrev_b16 v12.h, v12.l, 1
+; GFX1200-NEXT: v_add_nc_u16 v8.l, v10.h, -1
+; GFX1200-NEXT: v_and_b16 v10.h, v10.l, 1
+; GFX1200-NEXT: v_lshrrev_b16 v4.l, 8, v2.l
; GFX1200-NEXT: v_lshrrev_b16 v4.h, 6, v4.h
-; GFX1200-NEXT: v_add_nc_u16 v3.h, v6.h, v3.h
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, v8.l, v7.l
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v10.h
-; GFX1200-NEXT: v_lshlrev_b16 v10.h, v9.l, 1
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v6.l, v1.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v3.h
+; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v2.l
+; GFX1200-NEXT: v_and_b16 v8.l, v6.h, v8.l
+; GFX1200-NEXT: v_or_b16 v10.h, v11.l, v10.h
+; GFX1200-NEXT: v_lshrrev_b16 v11.l, v5.l, v6.h
; GFX1200-NEXT: v_and_b16 v4.l, v4.l, 0x80
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.l
-; GFX1200-NEXT: v_and_b16 v5.l, v11.h, 1
-; GFX1200-NEXT: v_add_nc_u16 v10.h, v10.h, -1
-; GFX1200-NEXT: v_sub_nc_u16 v9.h, 2, v6.l
-; GFX1200-NEXT: v_mov_b16_e32 v11.l, v12.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v8.l
+; GFX1200-NEXT: v_and_b16 v3.h, v3.h, v10.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1200-NEXT: v_and_b16 v8.l, v11.l, 1
+; GFX1200-NEXT: v_add_nc_u16 v10.h, v12.h, -1
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s0
-; GFX1200-NEXT: v_and_b16 v10.l, v7.l, v10.h
-; GFX1200-NEXT: v_and_b16 v10.h, v7.h, 1
-; GFX1200-NEXT: v_or_b16 v5.l, v11.l, v5.l
-; GFX1200-NEXT: v_mov_b16_e32 v11.l, v13.l
-; GFX1200-NEXT: v_lshrrev_b16 v7.l, v9.l, v7.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v10.l
-; GFX1200-NEXT: v_mov_b16_e32 v10.l, v12.l
-; GFX1200-NEXT: v_and_b16 v5.l, v8.h, v5.l
-; GFX1200-NEXT: v_and_b16 v8.h, v6.h, 1
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v9.l
+; GFX1200-NEXT: v_add_nc_u16 v3.h, v10.l, v3.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_mov_b16_e32 v9.l, v13.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, s1
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, v10.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v5.l, 0, v5.l, vcc_lo
-; GFX1200-NEXT: v_or_b16 v10.l, v11.l, v10.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v8.l
-; GFX1200-NEXT: v_mov_b16_e32 v12.l, v14.l
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, 6
-; GFX1200-NEXT: v_add_nc_u16 v5.l, v11.h, v5.l
-; GFX1200-NEXT: v_and_b16 v4.h, v4.h, v10.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_or_b16 v8.h, v12.l, v8.h
-; GFX1200-NEXT: v_lshlrev_b16 v8.l, 3, v3.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.l
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v7.h, v4.h
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v3.l
-; GFX1200-NEXT: v_and_b16 v7.l, v7.l, v8.h
-; GFX1200-NEXT: v_or_b16 v8.l, v4.l, v8.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, 8, vcc_lo
-; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, vcc_lo
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v4.h
+; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, s0
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.l
+; GFX1200-NEXT: v_lshrrev_b16 v5.l, v8.h, v6.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v3.h
+; GFX1200-NEXT: v_or_b16 v6.h, v9.l, v8.l
+; GFX1200-NEXT: v_and_b16 v8.l, v11.h, v10.h
+; GFX1200-NEXT: v_lshrrev_b16 v8.h, v7.l, v11.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, v7.l, s1
-; GFX1200-NEXT: v_or_b16 v3.h, v8.l, v3.h
-; GFX1200-NEXT: v_or_b16 v7.h, v4.l, v7.h
-; GFX1200-NEXT: v_lshrrev_b32_e32 v8, 31, v2
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v6.h, v7.l
+; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s1
+; GFX1200-NEXT: v_and_b16 v5.l, v5.l, v6.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v8.l
+; GFX1200-NEXT: v_and_b16 v9.l, v8.h, 1
; GFX1200-NEXT: v_min_u16 v6.h, v9.h, 15
-; GFX1200-NEXT: v_or_b16 v5.l, v7.h, v5.l
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v2.l
-; GFX1200-NEXT: v_mov_b16_e32 v7.l, v10.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 7, v3.l
-; GFX1200-NEXT: v_lshlrev_b16 v8.l, 7, v8.l
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v5.l, s0
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, vcc_lo
-; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.h, v7.l
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.h, v1.l
-; GFX1200-NEXT: v_sub_nc_u16 v7.l, v6.h, 1 clamp
+; GFX1200-NEXT: v_mov_b16_e32 v8.l, v10.l
+; GFX1200-NEXT: v_cndmask_b16 v5.l, 0, v5.l, s0
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, 8, s2
-; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v2.l, v2.l
-; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, 6
-; GFX1200-NEXT: v_and_b16 v8.h, v5.h, 0x3ff
-; GFX1200-NEXT: v_lshlrev_b16 v9.l, v7.l, 1
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s2
-; GFX1200-NEXT: v_or_b16 v7.h, v8.l, v7.h
-; GFX1200-NEXT: v_lshlrev_b16 v9.h, 3, v5.l
-; GFX1200-NEXT: v_or_b16 v10.l, v8.h, 0x400
-; GFX1200-NEXT: v_add_nc_u16 v9.l, v9.l, -1
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v4.l, s1
-; GFX1200-NEXT: v_or_b16 v3.l, v7.h, v3.l
-; GFX1200-NEXT: v_or_b16 v9.h, v8.l, v9.h
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v5.l
-; GFX1200-NEXT: v_and_b16 v2.l, v10.l, v9.l
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v8.h
-; GFX1200-NEXT: v_lshrrev_b16 v8.h, 8, v1.l
-; GFX1200-NEXT: v_or_b16 v4.l, v9.h, v4.h
-; GFX1200-NEXT: v_lshrrev_b16 v4.h, v6.h, v10.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v2.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v3.h, vcc_lo
-; GFX1200-NEXT: v_and_b16 v3.h, v5.h, 63
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s2
+; GFX1200-NEXT: v_lshrrev_b16 v9.h, v12.l, v11.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v7.l
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, v8.l
+; GFX1200-NEXT: v_add_nc_u16 v5.l, v11.l, v5.l
+; GFX1200-NEXT: v_mov_b16_e32 v10.l, v13.l
+; GFX1200-NEXT: v_and_b16 v8.l, v7.h, 1
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s1
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, 6
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v5.l
+; GFX1200-NEXT: v_or_b16 v9.l, v10.l, v9.l
+; GFX1200-NEXT: v_mov_b16_e32 v10.l, v14.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v4.l, v3.l, s0
-; GFX1200-NEXT: v_and_b16 v4.l, v4.h, 1
-; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, s1
-; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v2.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v3.h
-; GFX1200-NEXT: v_lshrrev_b16 v5.h, 6, v5.h
-; GFX1200-NEXT: v_and_b16 v8.h, v8.h, 0x80
-; GFX1200-NEXT: v_mov_b16_e32 v5.l, v9.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, s0
+; GFX1200-NEXT: v_and_b16 v7.l, v9.h, v9.l
+; GFX1200-NEXT: v_lshlrev_b16 v9.l, 3, v3.l
+; GFX1200-NEXT: v_or_b16 v8.l, v10.l, v8.l
+; GFX1200-NEXT: v_cndmask_b16 v9.h, 0, 8, s0
+; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v3.l
+; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, v7.l, s2
+; GFX1200-NEXT: v_or_b16 v9.l, v4.l, v9.l
+; GFX1200-NEXT: v_and_b16 v4.h, v4.h, v8.l
+; GFX1200-NEXT: v_or_b16 v9.h, v4.l, v9.h
+; GFX1200-NEXT: v_lshrrev_b32_e32 v10, 31, v2
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v8.h, v7.l
+; GFX1200-NEXT: v_or_b16 v3.h, v9.l, v3.h
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v7.h, v4.h
+; GFX1200-NEXT: v_or_b16 v5.l, v9.h, v5.l
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v7.l, v1.l
+; GFX1200-NEXT: v_sub_nc_u16 v7.h, v6.h, 1 clamp
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v3.l
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v5.l, s0
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.h
+; GFX1200-NEXT: v_and_b16 v5.l, v7.l, 0x3ff
+; GFX1200-NEXT: v_lshlrev_b16 v8.l, v7.h, 1
+; GFX1200-NEXT: v_cndmask_b16 v8.h, 0, 8, s1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v8.l, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v4.l, vcc_lo
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, s0
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v6.h
-; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v2.h, v2.h
-; GFX1200-NEXT: v_or_b16 v3.h, v5.l, v4.l
-; GFX1200-NEXT: v_lshrrev_b16 v4.l, v7.l, v10.l
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v7.l, v1.h
-; GFX1200-NEXT: v_and_b16 v5.l, v7.h, 1
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.l, 0x7f, v3.l, s0
-; GFX1200-NEXT: v_and_b16 v2.l, v2.l, 0xff
-; GFX1200-NEXT: v_and_b16 v3.h, v4.l, v3.h
-; GFX1200-NEXT: v_sub_nc_u16 v4.l, 2, v7.l
-; GFX1200-NEXT: v_or_b16 v5.l, v9.l, v5.l
-; GFX1200-NEXT: v_lshlrev_b16 v3.l, 8, v3.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.h, 0, v3.h, vcc_lo
-; GFX1200-NEXT: v_min_u16 v2.h, v4.l, 15
-; GFX1200-NEXT: v_and_b16 v4.l, v5.h, v5.l
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.l, v1.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_add_nc_u16 v3.h, v4.h, v3.h
-; GFX1200-NEXT: v_sub_nc_u16 v5.h, v2.h, 1 clamp
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_add_nc_u16 v4.l, v7.h, v4.l
-; GFX1200-NEXT: v_and_b16 v4.h, v5.l, 0x3ff
-; GFX1200-NEXT: v_and_b16 v9.l, v5.l, 63
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v3.h
-; GFX1200-NEXT: v_lshlrev_b16 v6.h, v5.h, 1
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.l
-; GFX1200-NEXT: v_or_b16 v7.h, v4.h, 0x400
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v9.l
-; GFX1200-NEXT: v_lshrrev_b16 v4.h, 7, v4.h
-; GFX1200-NEXT: v_add_nc_u16 v6.h, v6.h, -1
+; GFX1200-NEXT: v_lshlrev_b16 v9.h, 7, v10.l
+; GFX1200-NEXT: v_or_b16 v10.l, v5.l, 0x400
+; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, -1
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s1
+; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, v9.l
+; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v2.l, v2.l
+; GFX1200-NEXT: v_or_b16 v8.h, v9.h, v8.h
+; GFX1200-NEXT: v_and_b16 v4.l, v10.l, v8.l
+; GFX1200-NEXT: v_and_b16 v8.l, v7.l, 63
+; GFX1200-NEXT: v_add_nc_u16 v2.l, v5.h, 6
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, v6.h, v10.l
+; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s0
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v4.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v8.l
+; GFX1200-NEXT: v_lshlrev_b16 v4.l, 3, v2.l
+; GFX1200-NEXT: v_or_b16 v3.l, v8.h, v3.l
+; GFX1200-NEXT: v_and_b16 v8.h, v5.h, 1
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0
-; GFX1200-NEXT: v_lshrrev_b16 v9.h, v2.h, v7.h
; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s1
-; GFX1200-NEXT: v_lshrrev_b16 v5.l, 6, v5.l
-; GFX1200-NEXT: v_and_b16 v6.h, v7.h, v6.h
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, v10.l
-; GFX1200-NEXT: v_and_b16 v10.l, v9.h, 1
+; GFX1200-NEXT: v_lshrrev_b16 v5.l, 7, v5.l
+; GFX1200-NEXT: v_or_b16 v4.l, v9.h, v4.l
+; GFX1200-NEXT: v_lshrrev_b16 v7.h, v7.h, v10.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v6.h
+; GFX1200-NEXT: v_mov_b16_e32 v9.l, v11.l
+; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0
+; GFX1200-NEXT: v_or_b16 v4.l, v4.l, v4.h
+; GFX1200-NEXT: v_and_b16 v4.h, v5.l, 1
+; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v2.l
+; GFX1200-NEXT: v_or_b16 v8.l, v9.l, v8.h
; GFX1200-NEXT: v_mov_b16_e32 v9.l, v11.l
+; GFX1200-NEXT: v_lshrrev_b16 v6.h, 6, v7.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 8, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v6.h
-; GFX1200-NEXT: v_and_b16 v6.h, v4.h, 1
-; GFX1200-NEXT: v_lshrrev_b16 v5.h, v5.h, v7.h
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, 6
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v3.h, 0x7f, v3.h, vcc_lo
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s1
-; GFX1200-NEXT: v_or_b16 v6.h, v9.l, v6.h
-; GFX1200-NEXT: v_or_b16 v7.h, v8.h, v8.l
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v8.l, v0.l
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2.h
-; GFX1200-NEXT: v_mov_b16_e32 v11.l, v12.l
-; GFX1200-NEXT: v_and_b16 v5.l, v5.l, v6.h
-; GFX1200-NEXT: v_lshlrev_b16 v6.h, 3, v6.l
-; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, 0, s0
-; GFX1200-NEXT: v_or_b16 v3.h, v7.h, v3.h
-; GFX1200-NEXT: v_or_b16 v9.l, v11.l, v10.l
-; GFX1200-NEXT: v_add_nc_u16 v2.h, v4.h, v5.l
-; GFX1200-NEXT: v_sub_nc_u16 v5.l, 2, v8.l
-; GFX1200-NEXT: v_or_b16 v4.h, v8.h, v6.h
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v6.l
-; GFX1200-NEXT: v_and_b16 v5.h, v5.h, v9.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_min_u16 v5.l, v5.l, 15
-; GFX1200-NEXT: v_or_b16 v4.l, v4.h, v4.l
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v4.l, v3.l, s0
+; GFX1200-NEXT: v_and_b16 v2.l, v7.h, v8.l
+; GFX1200-NEXT: v_or_b16 v4.h, v9.l, v4.h
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v4.l, v1.h
+; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v2.h
+; GFX1200-NEXT: v_and_b16 v3.h, v3.h, 0xff
+; GFX1200-NEXT: v_cndmask_b16 v2.l, 0, v2.l, s1
+; GFX1200-NEXT: v_and_b16 v4.h, v6.h, v4.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, v5.h, vcc_lo
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v2.h
-; GFX1200-NEXT: v_sub_nc_u16 v6.h, v5.l, 1 clamp
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v4.l, v3.h, s0
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v1.l
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v9.h, v5.h
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v9.h, vcc_lo
+; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v2.h, v2.h
+; GFX1200-NEXT: v_add_nc_u16 v2.l, v5.h, v2.l
+; GFX1200-NEXT: v_sub_nc_u16 v5.h, 2, v4.l
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v5.l, v4.h
+; GFX1200-NEXT: v_lshrrev_b16 v5.l, 8, v1.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.h, v0.l
-; GFX1200-NEXT: v_lshlrev_b16 v7.h, v6.h, 1
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v3.l, 0x7f, v3.l, vcc_lo
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v2.l
+; GFX1200-NEXT: v_min_u16 v2.h, v5.h, 15
; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v4.h
-; GFX1200-NEXT: v_mov_b16_e32 v9.l, v10.l
-; GFX1200-NEXT: v_lshrrev_b32_e32 v10, 31, v1
-; GFX1200-NEXT: v_add_nc_u16 v7.h, v7.h, -1
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.h, v1.h
+; GFX1200-NEXT: v_and_b16 v5.l, v5.l, 0x80
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v8.h, s0
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s1
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v7.l, v9.l
-; GFX1200-NEXT: v_and_b16 v7.l, v5.h, 0x3ff
-; GFX1200-NEXT: v_cndmask_b16 v9.l, 0, 8, s1
-; GFX1200-NEXT: v_lshlrev_b16 v10.l, 7, v10.l
-; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v1.l, v1.l
+; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, 8, s0
+; GFX1200-NEXT: v_sub_nc_u16 v6.h, v2.h, 1 clamp
+; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, s1
+; GFX1200-NEXT: v_and_b16 v7.h, v5.h, 0x3ff
+; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, 0, s0
+; GFX1200-NEXT: v_or_b16 v7.l, v5.l, v7.l
+; GFX1200-NEXT: v_lshlrev_b16 v8.h, v6.h, 1
+; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, v8.l
+; GFX1200-NEXT: v_or_b16 v8.l, v7.h, 0x400
+; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v7.h
+; GFX1200-NEXT: v_or_b16 v2.l, v7.l, v2.l
+; GFX1200-NEXT: v_add_nc_u16 v8.h, v8.h, -1
+; GFX1200-NEXT: v_and_b16 v7.l, v5.h, 63
; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, 6
-; GFX1200-NEXT: v_or_b16 v9.h, v7.l, 0x400
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1200-NEXT: v_or_b16 v9.l, v10.l, v9.l
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.h, 0x7f, v3.h, s0
-; GFX1200-NEXT: v_lshlrev_b16 v4.l, 3, v6.l
-; GFX1200-NEXT: v_and_b16 v7.h, v9.h, v7.h
-; GFX1200-NEXT: v_lshrrev_b16 v10.h, v5.l, v9.h
-; GFX1200-NEXT: v_or_b16 v4.h, v9.l, v4.h
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v0.h
-; GFX1200-NEXT: v_or_b16 v4.l, v10.l, v4.l
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.h
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v7.h, v0.h
-; GFX1200-NEXT: v_and_b16 v3.h, v3.h, 0xff
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_or_b16 v2.h, v4.l, v2.h
+; GFX1200-NEXT: v_lshrrev_b16 v9.l, v2.h, v8.l
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, 6, v5.h
+; GFX1200-NEXT: v_and_b16 v8.h, v8.l, v8.h
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.l
+; GFX1200-NEXT: v_lshlrev_b16 v7.l, 3, v6.l
+; GFX1200-NEXT: v_and_b16 v9.h, v9.l, 1
+; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s1
+; GFX1200-NEXT: v_lshrrev_b16 v6.h, v6.h, v8.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v8.h
+; GFX1200-NEXT: v_and_b16 v8.h, v7.h, 1
+; GFX1200-NEXT: v_or_b16 v7.l, v5.l, v7.l
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v8.l, v0.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v2.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
+; GFX1200-NEXT: v_or_b16 v8.h, v10.l, v8.h
+; GFX1200-NEXT: v_or_b16 v4.h, v7.l, v4.h
; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v6.l
-; GFX1200-NEXT: v_and_b16 v4.l, v10.h, 1
-; GFX1200-NEXT: v_mov_b16_e32 v6.l, v11.l
+; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v1.l, v1.l
+; GFX1200-NEXT: v_or_b16 v9.h, v11.l, v9.h
+; GFX1200-NEXT: v_and_b16 v5.h, v5.h, v8.h
+; GFX1200-NEXT: v_lshlrev_b16 v3.l, 8, v3.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v4.h, vcc_lo
-; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v1.h
-; GFX1200-NEXT: v_sub_nc_u16 v4.h, 2, v7.h
-; GFX1200-NEXT: v_or_b16 v4.l, v6.l, v4.l
-; GFX1200-NEXT: v_lshrrev_b16 v6.l, v6.h, v9.h
+; GFX1200-NEXT: v_cndmask_b16 v2.l, v4.h, v2.l, vcc_lo
+; GFX1200-NEXT: v_and_b16 v6.l, v6.h, v9.h
+; GFX1200-NEXT: v_sub_nc_u16 v6.h, 2, v8.l
+; GFX1200-NEXT: v_add_nc_u16 v2.h, v7.h, v5.h
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, v6.l, s0
+; GFX1200-NEXT: v_min_u16 v4.h, v6.h, 15
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v2.h
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v0.l
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v1.l
+; GFX1200-NEXT: v_add_nc_u16 v5.h, v9.l, v5.h
+; GFX1200-NEXT: v_sub_nc_u16 v6.h, v4.h, 1 clamp
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, v2.h, v10.l, vcc_lo
-; GFX1200-NEXT: v_min_u16 v2.h, v4.h, 15
-; GFX1200-NEXT: v_and_b16 v4.h, v5.h, 63
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.l
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.l, v0.h
-; GFX1200-NEXT: v_and_b16 v4.l, v6.l, v4.l
-; GFX1200-NEXT: v_sub_nc_u16 v6.l, v2.h, 1 clamp
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v4.h
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, 7, v7.l
-; GFX1200-NEXT: v_and_b16 v9.l, v5.l, 63
-; GFX1200-NEXT: v_and_b16 v4.h, v5.l, 0x3ff
-; GFX1200-NEXT: v_lshlrev_b16 v7.l, v6.l, 1
+; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
+; GFX1200-NEXT: v_and_b16 v7.h, v6.l, 0x3ff
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s1
-; GFX1200-NEXT: v_and_b16 v8.h, v6.h, 1
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v9.l
-; GFX1200-NEXT: v_or_b16 v9.h, v4.h, 0x400
-; GFX1200-NEXT: v_add_nc_u16 v7.l, v7.l, -1
-; GFX1200-NEXT: v_mov_b16_e32 v10.l, v11.l
-; GFX1200-NEXT: v_lshrrev_b16 v4.h, 7, v4.h
+; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v5.l, s0
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v5.h
+; GFX1200-NEXT: v_lshlrev_b16 v8.h, v6.h, 1
+; GFX1200-NEXT: v_add_nc_u16 v4.l, v4.l, v7.l
+; GFX1200-NEXT: v_or_b16 v5.l, v7.h, 0x400
+; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v1
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, vcc_lo
+; GFX1200-NEXT: v_add_nc_u16 v7.l, v8.h, -1
+; GFX1200-NEXT: v_add_nc_u16 v1.l, v4.l, 6
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v4.l, v5.h, 0, s0
+; GFX1200-NEXT: v_lshlrev_b16 v8.h, 7, v9.l
+; GFX1200-NEXT: v_lshrrev_b16 v9.h, v4.h, v5.l
+; GFX1200-NEXT: v_and_b16 v5.h, v5.l, v7.l
+; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, 8, s0
+; GFX1200-NEXT: v_lshlrev_b16 v9.l, 3, v1.l
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v1.l
+; GFX1200-NEXT: v_lshrrev_b16 v5.l, v6.h, v5.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.h
+; GFX1200-NEXT: v_or_b16 v5.h, v8.h, v7.l
+; GFX1200-NEXT: v_or_b16 v7.l, v8.h, v9.l
+; GFX1200-NEXT: v_and_b16 v9.l, v9.h, 1
+; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v2.l, s1
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s1
-; GFX1200-NEXT: v_lshrrev_b16 v5.h, 6, v5.h
-; GFX1200-NEXT: v_and_b16 v7.l, v9.h, v7.l
-; GFX1200-NEXT: v_or_b16 v8.h, v10.l, v8.h
-; GFX1200-NEXT: v_and_b16 v9.l, v4.h, 1
-; GFX1200-NEXT: v_mov_b16_e32 v10.l, v11.l
-; GFX1200-NEXT: v_lshrrev_b16 v5.l, 6, v5.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v7.l
-; GFX1200-NEXT: v_lshrrev_b16 v11.l, v2.h, v9.h
-; GFX1200-NEXT: v_and_b16 v5.h, v5.h, v8.h
-; GFX1200-NEXT: v_or_b16 v7.l, v10.l, v9.l
-; GFX1200-NEXT: v_lshrrev_b16 v6.l, v6.l, v9.h
+; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v10.h, v0.h
+; GFX1200-NEXT: v_or_b16 v2.h, v7.l, v2.h
+; GFX1200-NEXT: v_or_b16 v5.h, v5.h, v4.l
+; GFX1200-NEXT: v_or_b16 v4.l, v3.h, v3.l
+; GFX1200-NEXT: v_or_b16 v1.l, v10.l, v9.l
+; GFX1200-NEXT: v_sub_nc_u16 v7.l, 2, v10.h
+; GFX1200-NEXT: v_and_b16 v3.h, v6.l, 63
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v5.h, vcc_lo
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v4.h
+; GFX1200-NEXT: v_and_b16 v1.l, v5.l, v1.l
+; GFX1200-NEXT: v_min_u16 v3.l, v7.l, 15
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v4.h, v0.h
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, 7, v7.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v3.h
+; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l
+; GFX1200-NEXT: v_sub_nc_u16 v5.l, v3.l, 1 clamp
+; GFX1200-NEXT: v_and_b16 v3.h, v4.h, 0x3ff
+; GFX1200-NEXT: v_and_b16 v7.h, v4.h, 63
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s1
-; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v1.h, v1.h
+; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0
+; GFX1200-NEXT: v_and_b16 v7.l, v5.h, 1
+; GFX1200-NEXT: v_lshlrev_b16 v6.h, v5.l, 1
+; GFX1200-NEXT: v_or_b16 v10.l, v3.h, 0x400
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.h
+; GFX1200-NEXT: v_mov_b16_e32 v9.l, v11.l
+; GFX1200-NEXT: v_lshrrev_b16 v3.h, 7, v3.h
+; GFX1200-NEXT: v_add_nc_u16 v6.h, v6.h, -1
+; GFX1200-NEXT: v_lshrrev_b16 v4.h, 6, v4.h
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0
+; GFX1200-NEXT: v_or_b16 v7.l, v9.l, v7.l
+; GFX1200-NEXT: v_and_b16 v7.h, v3.h, 1
+; GFX1200-NEXT: v_and_b16 v6.h, v10.l, v6.h
+; GFX1200-NEXT: v_lshrrev_b16 v5.l, v5.l, v10.l
+; GFX1200-NEXT: v_mov_b16_e32 v9.l, v11.l
+; GFX1200-NEXT: v_lshrrev_b16 v11.l, v3.l, v10.l
+; GFX1200-NEXT: v_and_b16 v6.l, v6.l, v7.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.h, 0, v4.l, vcc_lo
-; GFX1200-NEXT: v_and_b16 v4.l, v5.l, v7.l
-; GFX1200-NEXT: v_and_b16 v8.h, v11.l, 1
+; GFX1200-NEXT: v_cndmask_b16 v1.l, 0, v1.l, vcc_lo
+; GFX1200-NEXT: v_or_b16 v6.h, v9.l, v7.h
+; GFX1200-NEXT: v_and_b16 v7.l, v11.l, 1
+; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, v6.l
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v3.l
+; GFX1200-NEXT: v_and_b16 v4.h, v4.h, v6.h
+; GFX1200-NEXT: v_add_nc_u16 v1.l, v9.h, v1.l
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.h
; GFX1200-NEXT: v_mov_b16_e32 v9.l, v12.l
-; GFX1200-NEXT: v_add_nc_u16 v5.l, v6.h, v5.h
-; GFX1200-NEXT: v_add_nc_u16 v1.h, v10.h, v1.h
-; GFX1200-NEXT: v_add_nc_u16 v4.l, v4.h, v4.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v2.h
-; GFX1200-NEXT: v_or_b16 v5.h, v9.l, v8.h
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 7, v1.h
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s4, 7, v4.l
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7f, v1.l, s1
-; GFX1200-NEXT: v_and_b16 v4.h, v6.l, v5.h
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v1.h
+; GFX1200-NEXT: v_add_nc_u16 v3.h, v3.h, v4.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 7, v1.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, 0, s3
-; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s4
-; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, vcc_lo
-; GFX1200-NEXT: v_cndmask_b16 v2.h, 0, v4.h, s2
-; GFX1200-NEXT: v_mov_b16_e32 v6.l, v9.l
-; GFX1200-NEXT: v_lshrrev_b16 v4.h, 8, v0.l
-; GFX1200-NEXT: v_mov_b16_e32 v7.l, v10.l
-; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, 0, s4
-; GFX1200-NEXT: v_add_nc_u16 v2.h, v11.l, v2.h
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v8.l, v6.l
-; GFX1200-NEXT: v_lshrrev_b32_e32 v8, 31, v0
-; GFX1200-NEXT: v_add_nc_u16 v6.h, v7.h, v7.l
-; GFX1200-NEXT: v_cndmask_b16 v6.l, 0, 8, s3
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 7, v2.h
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, 6
-; GFX1200-NEXT: v_and_b16 v4.h, v4.h, 0x80
-; GFX1200-NEXT: v_add_nc_u16 v6.h, v6.h, 6
-; GFX1200-NEXT: v_lshlrev_b16 v8.l, 7, v8.l
+; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, vcc_lo
+; GFX1200-NEXT: v_or_b16 v6.l, v9.l, v7.l
+; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 7, v3.h
+; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v0
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, 8, s2
-; GFX1200-NEXT: v_lshlrev_b16 v7.l, 3, v5.h
-; GFX1200-NEXT: v_or_b16 v6.l, v4.h, v6.l
-; GFX1200-NEXT: v_lshlrev_b16 v8.h, 3, v6.h
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, s2
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v8.h, s0
+; GFX1200-NEXT: v_and_b16 v4.h, v5.l, v6.l
+; GFX1200-NEXT: v_mov_b16_e32 v3.l, v7.l
+; GFX1200-NEXT: v_cndmask_b32_e64 v6, 0, 1, s3
+; GFX1200-NEXT: v_lshrrev_b16 v5.l, 8, v0.l
+; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 8, s2
+; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v4.h, s1
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v8.l, v3.l
+; GFX1200-NEXT: v_add_nc_u16 v6.l, v10.h, v6.l
+; GFX1200-NEXT: v_lshlrev_b16 v8.l, 7, v9.l
+; GFX1200-NEXT: v_and_b16 v5.l, v5.l, 0x80
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v11.l, v4.h
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, 6
+; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, 6
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s3
+; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v1.h, v1.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v4.h
+; GFX1200-NEXT: v_lshlrev_b16 v7.l, 3, v3.l
+; GFX1200-NEXT: v_lshlrev_b16 v9.l, 3, v6.l
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, 0, s2
+; GFX1200-NEXT: v_or_b16 v6.h, v5.l, v6.h
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, 8, s1
+; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s1
+; GFX1200-NEXT: v_or_b16 v9.l, v8.l, v9.l
+; GFX1200-NEXT: v_or_b16 v7.l, v5.l, v7.l
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v3.l
; GFX1200-NEXT: v_or_b16 v7.h, v8.l, v7.h
-; GFX1200-NEXT: v_or_b16 v7.l, v4.h, v7.l
-; GFX1200-NEXT: v_or_b16 v1.h, v6.l, v1.h
-; GFX1200-NEXT: v_or_b16 v8.h, v8.l, v8.h
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v5.h
-; GFX1200-NEXT: v_or_b16 v2.h, v7.h, v2.h
-; GFX1200-NEXT: v_or_b16 v5.l, v7.l, v5.l
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s2, 1, v6.h
-; GFX1200-NEXT: v_or_b16 v4.l, v8.h, v4.l
-; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v6.l
+; GFX1200-NEXT: v_or_b16 v3.h, v9.l, v3.h
+; GFX1200-NEXT: v_cndmask_b16 v2.h, 0x7f, v2.h, s0
+; GFX1200-NEXT: v_or_b16 v1.l, v6.h, v1.l
+; GFX1200-NEXT: v_or_b16 v3.l, v7.h, v4.h
+; GFX1200-NEXT: v_or_b16 v5.h, v7.l, v5.h
+; GFX1200-NEXT: v_and_b16 v2.l, v2.l, 0xff
+; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v0.h, v0.h
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v1.h, v3.h, v3.l, s1
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v0.h
+; GFX1200-NEXT: v_lshlrev_b16 v0.h, 8, v2.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.h, v5.l, v1.h, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v5.h, v1.l, vcc_lo
; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v0.l
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v4.l, v2.h, s2
-; GFX1200-NEXT: v_or_b16 v4.l, v2.l, v3.l
+; GFX1200-NEXT: v_cvt_u32_u16_e32 v3, v4.l
+; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v0.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, v4.h, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, vcc_lo
; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v0.l
-; GFX1200-NEXT: v_cndmask_b16 v0.l, v2.h, v8.l, s0
-; GFX1200-NEXT: v_or_b16 v2.h, v3.h, v1.l
-; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v0.h, v0.h
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v0.h, 0x7f, v1.h, vcc_lo
-; GFX1200-NEXT: v_cvt_u32_u16_e32 v1, v4.l
-; GFX1200-NEXT: v_mov_b16_e32 v3.l, v2.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v0.l, v1.h, v8.l, s1
+; GFX1200-NEXT: v_or_b32_e32 v6, 0x7f7f0000, v3
+; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX1200-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7f, v1.l, vcc_lo
; GFX1200-NEXT: v_cndmask_b16 v0.l, 0x7f, v0.l, s0
-; GFX1200-NEXT: v_and_b16 v0.h, v0.h, 0xff
-; GFX1200-NEXT: v_or_b32_e32 v6, 0x7f7f0000, v1
-; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_lshlrev_b16 v2.l, 8, v0.l
; GFX1200-NEXT: v_lshrrev_b64 v[5:6], 24, v[5:6]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_or_b16 v0.l, v0.h, v2.l
-; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX1200-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_and_b16 v0.h, v1.l, 0xff
+; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v0.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX1200-NEXT: v_or_b16 v0.l, v0.h, v1.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-TRUE16-LABEL: to_fp8_v5f16:
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v3.h, v1
+; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v3.l, v1
; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v5.h, s0
; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v5.l, v2
-; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v3.l, v0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.h
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_dual_lshlrev_b32 v4, 16, v1 :: v_dual_lshrrev_b32 v1, 8, v3
+; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v0.l, v0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
+; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1250-TRUE16-NEXT: v_lshrrev_b64 v[6:7], 24, v[4:5]
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.l
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
@@ -1390,404 +1378,392 @@ define <6 x i8> @to_fp8_v6f16(<6 x half> %x) {
; GFX1200-NEXT: s_wait_kmcnt 0x0
; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v3.l, v2.l
; GFX1200-NEXT: v_frexp_mant_f16_e32 v3.h, v2.l
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v4.h, v2.h
; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v5.h, v2.h
-; GFX1200-NEXT: v_lshrrev_b16 v4.l, 8, v2.l
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v4.h, v2.h
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v6.l, v1.l
; GFX1200-NEXT: v_sub_nc_u16 v5.l, 2, v3.l
; GFX1200-NEXT: v_and_b16 v6.h, v3.h, 0x3ff
; GFX1200-NEXT: v_and_b16 v7.l, v3.h, 63
; GFX1200-NEXT: v_sub_nc_u16 v8.l, 2, v5.h
-; GFX1200-NEXT: v_and_b16 v9.l, v4.h, 63
-; GFX1200-NEXT: v_min_u16 v5.l, v5.l, 15
-; GFX1200-NEXT: v_or_b16 v10.l, v6.h, 0x400
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, 7, v6.h
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.l
; GFX1200-NEXT: v_lshrrev_b16 v3.h, 6, v3.h
+; GFX1200-NEXT: v_min_u16 v5.l, v5.l, 15
+; GFX1200-NEXT: v_lshrrev_b16 v10.l, 7, v6.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.l
+; GFX1200-NEXT: v_or_b16 v6.h, v6.h, 0x400
+; GFX1200-NEXT: v_min_u16 v7.l, v8.l, 15
; GFX1200-NEXT: v_sub_nc_u16 v8.h, v5.l, 1 clamp
-; GFX1200-NEXT: v_min_u16 v8.l, v8.l, 15
-; GFX1200-NEXT: v_and_b16 v11.l, v6.h, 1
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v9.l
-; GFX1200-NEXT: v_lshlrev_b16 v10.h, v8.h, 1
-; GFX1200-NEXT: v_sub_nc_u16 v9.l, v8.l, 1 clamp
; GFX1200-NEXT: v_and_b16 v7.h, v4.h, 0x3ff
-; GFX1200-NEXT: v_or_b16 v11.l, v12.l, v11.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc_lo
-; GFX1200-NEXT: v_add_nc_u16 v10.h, v10.h, -1
-; GFX1200-NEXT: v_lshrrev_b16 v11.h, v5.l, v10.l
-; GFX1200-NEXT: v_or_b16 v7.l, v7.h, 0x400
-; GFX1200-NEXT: v_and_b16 v3.h, v3.h, v11.l
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0
+; GFX1200-NEXT: v_and_b16 v9.l, v4.h, 63
+; GFX1200-NEXT: v_sub_nc_u16 v12.l, v7.l, 1 clamp
+; GFX1200-NEXT: v_lshlrev_b16 v10.h, v8.h, 1
+; GFX1200-NEXT: v_or_b16 v11.h, v7.h, 0x400
+; GFX1200-NEXT: v_sub_nc_u16 v9.h, 2, v6.l
; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v7.h
-; GFX1200-NEXT: v_and_b16 v10.h, v10.l, v10.h
-; GFX1200-NEXT: v_lshrrev_b16 v8.h, v8.h, v10.l
+; GFX1200-NEXT: v_lshlrev_b16 v12.h, v12.l, 1
+; GFX1200-NEXT: v_add_nc_u16 v8.l, v10.h, -1
+; GFX1200-NEXT: v_and_b16 v10.h, v10.l, 1
+; GFX1200-NEXT: v_lshrrev_b16 v4.l, 8, v2.l
; GFX1200-NEXT: v_lshrrev_b16 v4.h, 6, v4.h
-; GFX1200-NEXT: v_add_nc_u16 v3.h, v6.h, v3.h
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, v8.l, v7.l
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v10.h
-; GFX1200-NEXT: v_lshlrev_b16 v10.h, v9.l, 1
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v6.l, v1.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v3.h
+; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v2.l
+; GFX1200-NEXT: v_and_b16 v8.l, v6.h, v8.l
+; GFX1200-NEXT: v_or_b16 v10.h, v11.l, v10.h
+; GFX1200-NEXT: v_lshrrev_b16 v11.l, v5.l, v6.h
; GFX1200-NEXT: v_and_b16 v4.l, v4.l, 0x80
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.l
-; GFX1200-NEXT: v_and_b16 v5.l, v11.h, 1
-; GFX1200-NEXT: v_add_nc_u16 v10.h, v10.h, -1
-; GFX1200-NEXT: v_sub_nc_u16 v9.h, 2, v6.l
-; GFX1200-NEXT: v_mov_b16_e32 v11.l, v12.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v8.l
+; GFX1200-NEXT: v_and_b16 v3.h, v3.h, v10.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1200-NEXT: v_and_b16 v8.l, v11.l, 1
+; GFX1200-NEXT: v_add_nc_u16 v10.h, v12.h, -1
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s0
-; GFX1200-NEXT: v_and_b16 v10.l, v7.l, v10.h
-; GFX1200-NEXT: v_and_b16 v10.h, v7.h, 1
-; GFX1200-NEXT: v_or_b16 v5.l, v11.l, v5.l
-; GFX1200-NEXT: v_mov_b16_e32 v11.l, v13.l
-; GFX1200-NEXT: v_lshrrev_b16 v7.l, v9.l, v7.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v10.l
-; GFX1200-NEXT: v_mov_b16_e32 v10.l, v12.l
-; GFX1200-NEXT: v_and_b16 v5.l, v8.h, v5.l
-; GFX1200-NEXT: v_and_b16 v8.h, v6.h, 1
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v9.l
+; GFX1200-NEXT: v_add_nc_u16 v3.h, v10.l, v3.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_mov_b16_e32 v9.l, v13.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, s1
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, v10.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v5.l, 0, v5.l, vcc_lo
-; GFX1200-NEXT: v_or_b16 v10.l, v11.l, v10.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v8.l
-; GFX1200-NEXT: v_mov_b16_e32 v12.l, v14.l
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, 6
-; GFX1200-NEXT: v_add_nc_u16 v5.l, v11.h, v5.l
-; GFX1200-NEXT: v_and_b16 v4.h, v4.h, v10.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_or_b16 v8.h, v12.l, v8.h
-; GFX1200-NEXT: v_lshlrev_b16 v8.l, 3, v3.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.l
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v7.h, v4.h
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v3.l
-; GFX1200-NEXT: v_and_b16 v7.l, v7.l, v8.h
-; GFX1200-NEXT: v_or_b16 v8.l, v4.l, v8.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, 8, vcc_lo
-; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, vcc_lo
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v4.h
+; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, s0
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.l
+; GFX1200-NEXT: v_lshrrev_b16 v5.l, v8.h, v6.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v3.h
+; GFX1200-NEXT: v_or_b16 v6.h, v9.l, v8.l
+; GFX1200-NEXT: v_and_b16 v8.l, v11.h, v10.h
+; GFX1200-NEXT: v_lshrrev_b16 v8.h, v7.l, v11.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, v7.l, s1
-; GFX1200-NEXT: v_or_b16 v3.h, v8.l, v3.h
-; GFX1200-NEXT: v_or_b16 v7.h, v4.l, v7.h
-; GFX1200-NEXT: v_lshrrev_b32_e32 v8, 31, v2
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v6.h, v7.l
+; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s1
+; GFX1200-NEXT: v_and_b16 v5.l, v5.l, v6.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v8.l
+; GFX1200-NEXT: v_and_b16 v9.l, v8.h, 1
; GFX1200-NEXT: v_min_u16 v6.h, v9.h, 15
-; GFX1200-NEXT: v_or_b16 v5.l, v7.h, v5.l
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v2.l
-; GFX1200-NEXT: v_mov_b16_e32 v7.l, v10.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 7, v3.l
-; GFX1200-NEXT: v_lshlrev_b16 v8.l, 7, v8.l
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v5.l, s0
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, vcc_lo
-; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.h, v7.l
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.h, v1.l
-; GFX1200-NEXT: v_sub_nc_u16 v7.l, v6.h, 1 clamp
+; GFX1200-NEXT: v_mov_b16_e32 v8.l, v10.l
+; GFX1200-NEXT: v_cndmask_b16 v5.l, 0, v5.l, s0
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, 8, s2
-; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v2.l, v2.l
-; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, 6
-; GFX1200-NEXT: v_and_b16 v8.h, v5.h, 0x3ff
-; GFX1200-NEXT: v_lshlrev_b16 v9.l, v7.l, 1
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s2
-; GFX1200-NEXT: v_or_b16 v7.h, v8.l, v7.h
-; GFX1200-NEXT: v_lshlrev_b16 v9.h, 3, v5.l
-; GFX1200-NEXT: v_or_b16 v10.l, v8.h, 0x400
-; GFX1200-NEXT: v_add_nc_u16 v9.l, v9.l, -1
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v4.l, s1
-; GFX1200-NEXT: v_or_b16 v3.l, v7.h, v3.l
-; GFX1200-NEXT: v_or_b16 v9.h, v8.l, v9.h
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v5.l
-; GFX1200-NEXT: v_and_b16 v2.l, v10.l, v9.l
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v8.h
-; GFX1200-NEXT: v_lshrrev_b16 v8.h, 8, v1.l
-; GFX1200-NEXT: v_or_b16 v4.l, v9.h, v4.h
-; GFX1200-NEXT: v_lshrrev_b16 v4.h, v6.h, v10.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v2.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v3.h, vcc_lo
-; GFX1200-NEXT: v_and_b16 v3.h, v5.h, 63
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s2
+; GFX1200-NEXT: v_lshrrev_b16 v9.h, v12.l, v11.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v7.l
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, v8.l
+; GFX1200-NEXT: v_add_nc_u16 v5.l, v11.l, v5.l
+; GFX1200-NEXT: v_mov_b16_e32 v10.l, v13.l
+; GFX1200-NEXT: v_and_b16 v8.l, v7.h, 1
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s1
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, 6
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v5.l
+; GFX1200-NEXT: v_or_b16 v9.l, v10.l, v9.l
+; GFX1200-NEXT: v_mov_b16_e32 v10.l, v14.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v4.l, v3.l, s0
-; GFX1200-NEXT: v_and_b16 v4.l, v4.h, 1
-; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, s1
-; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v2.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v3.h
-; GFX1200-NEXT: v_lshrrev_b16 v5.h, 6, v5.h
-; GFX1200-NEXT: v_and_b16 v8.h, v8.h, 0x80
-; GFX1200-NEXT: v_mov_b16_e32 v5.l, v9.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, s0
+; GFX1200-NEXT: v_and_b16 v7.l, v9.h, v9.l
+; GFX1200-NEXT: v_lshlrev_b16 v9.l, 3, v3.l
+; GFX1200-NEXT: v_or_b16 v8.l, v10.l, v8.l
+; GFX1200-NEXT: v_cndmask_b16 v9.h, 0, 8, s0
+; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v3.l
+; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, v7.l, s2
+; GFX1200-NEXT: v_or_b16 v9.l, v4.l, v9.l
+; GFX1200-NEXT: v_and_b16 v4.h, v4.h, v8.l
+; GFX1200-NEXT: v_or_b16 v9.h, v4.l, v9.h
+; GFX1200-NEXT: v_lshrrev_b32_e32 v10, 31, v2
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v8.h, v7.l
+; GFX1200-NEXT: v_or_b16 v3.h, v9.l, v3.h
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v7.h, v4.h
+; GFX1200-NEXT: v_or_b16 v5.l, v9.h, v5.l
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v7.l, v1.l
+; GFX1200-NEXT: v_sub_nc_u16 v7.h, v6.h, 1 clamp
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v3.l
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v5.l, s0
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.h
+; GFX1200-NEXT: v_and_b16 v5.l, v7.l, 0x3ff
+; GFX1200-NEXT: v_lshlrev_b16 v8.l, v7.h, 1
+; GFX1200-NEXT: v_cndmask_b16 v8.h, 0, 8, s1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v8.l, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v4.l, vcc_lo
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, s0
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v6.h
-; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v2.h, v2.h
-; GFX1200-NEXT: v_or_b16 v3.h, v5.l, v4.l
-; GFX1200-NEXT: v_lshrrev_b16 v4.l, v7.l, v10.l
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v7.l, v1.h
-; GFX1200-NEXT: v_and_b16 v5.l, v7.h, 1
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.l, 0x7f, v3.l, s0
-; GFX1200-NEXT: v_and_b16 v2.l, v2.l, 0xff
-; GFX1200-NEXT: v_and_b16 v3.h, v4.l, v3.h
-; GFX1200-NEXT: v_sub_nc_u16 v4.l, 2, v7.l
-; GFX1200-NEXT: v_or_b16 v5.l, v9.l, v5.l
-; GFX1200-NEXT: v_lshlrev_b16 v3.l, 8, v3.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.h, 0, v3.h, vcc_lo
-; GFX1200-NEXT: v_min_u16 v2.h, v4.l, 15
-; GFX1200-NEXT: v_and_b16 v4.l, v5.h, v5.l
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.l, v1.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_add_nc_u16 v3.h, v4.h, v3.h
-; GFX1200-NEXT: v_sub_nc_u16 v5.h, v2.h, 1 clamp
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_add_nc_u16 v4.l, v7.h, v4.l
-; GFX1200-NEXT: v_and_b16 v4.h, v5.l, 0x3ff
-; GFX1200-NEXT: v_and_b16 v9.l, v5.l, 63
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v3.h
-; GFX1200-NEXT: v_lshlrev_b16 v6.h, v5.h, 1
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.l
-; GFX1200-NEXT: v_or_b16 v7.h, v4.h, 0x400
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v9.l
-; GFX1200-NEXT: v_lshrrev_b16 v4.h, 7, v4.h
-; GFX1200-NEXT: v_add_nc_u16 v6.h, v6.h, -1
+; GFX1200-NEXT: v_lshlrev_b16 v9.h, 7, v10.l
+; GFX1200-NEXT: v_or_b16 v10.l, v5.l, 0x400
+; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, -1
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s1
+; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, v9.l
+; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v2.l, v2.l
+; GFX1200-NEXT: v_or_b16 v8.h, v9.h, v8.h
+; GFX1200-NEXT: v_and_b16 v4.l, v10.l, v8.l
+; GFX1200-NEXT: v_and_b16 v8.l, v7.l, 63
+; GFX1200-NEXT: v_add_nc_u16 v2.l, v5.h, 6
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, v6.h, v10.l
+; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s0
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v4.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v8.l
+; GFX1200-NEXT: v_lshlrev_b16 v4.l, 3, v2.l
+; GFX1200-NEXT: v_or_b16 v3.l, v8.h, v3.l
+; GFX1200-NEXT: v_and_b16 v8.h, v5.h, 1
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0
-; GFX1200-NEXT: v_lshrrev_b16 v9.h, v2.h, v7.h
; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s1
-; GFX1200-NEXT: v_lshrrev_b16 v5.l, 6, v5.l
-; GFX1200-NEXT: v_and_b16 v6.h, v7.h, v6.h
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, v10.l
-; GFX1200-NEXT: v_and_b16 v10.l, v9.h, 1
+; GFX1200-NEXT: v_lshrrev_b16 v5.l, 7, v5.l
+; GFX1200-NEXT: v_or_b16 v4.l, v9.h, v4.l
+; GFX1200-NEXT: v_lshrrev_b16 v7.h, v7.h, v10.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v6.h
; GFX1200-NEXT: v_mov_b16_e32 v9.l, v11.l
+; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0
+; GFX1200-NEXT: v_or_b16 v4.l, v4.l, v4.h
+; GFX1200-NEXT: v_and_b16 v4.h, v5.l, 1
+; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v2.l
+; GFX1200-NEXT: v_or_b16 v8.l, v9.l, v8.h
+; GFX1200-NEXT: v_mov_b16_e32 v9.l, v11.l
+; GFX1200-NEXT: v_lshrrev_b16 v6.h, 6, v7.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 8, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v6.h
-; GFX1200-NEXT: v_and_b16 v6.h, v4.h, 1
-; GFX1200-NEXT: v_lshrrev_b16 v5.h, v5.h, v7.h
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, 6
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v3.h, 0x7f, v3.h, vcc_lo
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s1
-; GFX1200-NEXT: v_or_b16 v6.h, v9.l, v6.h
-; GFX1200-NEXT: v_or_b16 v7.h, v8.h, v8.l
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v8.l, v0.l
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2.h
-; GFX1200-NEXT: v_mov_b16_e32 v11.l, v12.l
-; GFX1200-NEXT: v_and_b16 v5.l, v5.l, v6.h
-; GFX1200-NEXT: v_lshlrev_b16 v6.h, 3, v6.l
-; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, 0, s0
-; GFX1200-NEXT: v_or_b16 v3.h, v7.h, v3.h
-; GFX1200-NEXT: v_or_b16 v9.l, v11.l, v10.l
-; GFX1200-NEXT: v_add_nc_u16 v2.h, v4.h, v5.l
-; GFX1200-NEXT: v_sub_nc_u16 v5.l, 2, v8.l
-; GFX1200-NEXT: v_or_b16 v4.h, v8.h, v6.h
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v6.l
-; GFX1200-NEXT: v_and_b16 v5.h, v5.h, v9.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_min_u16 v5.l, v5.l, 15
-; GFX1200-NEXT: v_or_b16 v4.l, v4.h, v4.l
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v4.l, v3.l, s0
+; GFX1200-NEXT: v_and_b16 v2.l, v7.h, v8.l
+; GFX1200-NEXT: v_or_b16 v4.h, v9.l, v4.h
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v4.l, v1.h
+; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v2.h
+; GFX1200-NEXT: v_and_b16 v3.h, v3.h, 0xff
+; GFX1200-NEXT: v_cndmask_b16 v2.l, 0, v2.l, s1
+; GFX1200-NEXT: v_and_b16 v4.h, v6.h, v4.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, v5.h, vcc_lo
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v2.h
-; GFX1200-NEXT: v_sub_nc_u16 v6.h, v5.l, 1 clamp
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v4.l, v3.h, s0
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v1.l
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v9.h, v5.h
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v9.h, vcc_lo
+; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v2.h, v2.h
+; GFX1200-NEXT: v_add_nc_u16 v2.l, v5.h, v2.l
+; GFX1200-NEXT: v_sub_nc_u16 v5.h, 2, v4.l
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v5.l, v4.h
+; GFX1200-NEXT: v_lshrrev_b16 v5.l, 8, v1.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.h, v0.l
-; GFX1200-NEXT: v_lshlrev_b16 v7.h, v6.h, 1
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v3.l, 0x7f, v3.l, vcc_lo
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v2.l
+; GFX1200-NEXT: v_min_u16 v2.h, v5.h, 15
; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v4.h
-; GFX1200-NEXT: v_mov_b16_e32 v9.l, v10.l
-; GFX1200-NEXT: v_lshrrev_b32_e32 v10, 31, v1
-; GFX1200-NEXT: v_add_nc_u16 v7.h, v7.h, -1
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.h, v1.h
+; GFX1200-NEXT: v_and_b16 v5.l, v5.l, 0x80
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v8.h, s0
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s1
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v7.l, v9.l
-; GFX1200-NEXT: v_and_b16 v7.l, v5.h, 0x3ff
-; GFX1200-NEXT: v_cndmask_b16 v9.l, 0, 8, s1
-; GFX1200-NEXT: v_lshlrev_b16 v10.l, 7, v10.l
-; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v1.l, v1.l
+; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, 8, s0
+; GFX1200-NEXT: v_sub_nc_u16 v6.h, v2.h, 1 clamp
+; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, s1
+; GFX1200-NEXT: v_and_b16 v7.h, v5.h, 0x3ff
+; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, 0, s0
+; GFX1200-NEXT: v_or_b16 v7.l, v5.l, v7.l
+; GFX1200-NEXT: v_lshlrev_b16 v8.h, v6.h, 1
+; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, v8.l
+; GFX1200-NEXT: v_or_b16 v8.l, v7.h, 0x400
+; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v7.h
+; GFX1200-NEXT: v_or_b16 v2.l, v7.l, v2.l
+; GFX1200-NEXT: v_add_nc_u16 v8.h, v8.h, -1
+; GFX1200-NEXT: v_and_b16 v7.l, v5.h, 63
; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, 6
-; GFX1200-NEXT: v_or_b16 v9.h, v7.l, 0x400
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1200-NEXT: v_or_b16 v9.l, v10.l, v9.l
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.h, 0x7f, v3.h, s0
-; GFX1200-NEXT: v_lshlrev_b16 v4.l, 3, v6.l
-; GFX1200-NEXT: v_and_b16 v7.h, v9.h, v7.h
-; GFX1200-NEXT: v_lshrrev_b16 v10.h, v5.l, v9.h
-; GFX1200-NEXT: v_or_b16 v4.h, v9.l, v4.h
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v0.h
-; GFX1200-NEXT: v_or_b16 v4.l, v10.l, v4.l
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.h
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v7.h, v0.h
-; GFX1200-NEXT: v_and_b16 v3.h, v3.h, 0xff
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_or_b16 v2.h, v4.l, v2.h
+; GFX1200-NEXT: v_lshrrev_b16 v9.l, v2.h, v8.l
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, 6, v5.h
+; GFX1200-NEXT: v_and_b16 v8.h, v8.l, v8.h
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.l
+; GFX1200-NEXT: v_lshlrev_b16 v7.l, 3, v6.l
+; GFX1200-NEXT: v_and_b16 v9.h, v9.l, 1
+; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s1
+; GFX1200-NEXT: v_lshrrev_b16 v6.h, v6.h, v8.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v8.h
+; GFX1200-NEXT: v_and_b16 v8.h, v7.h, 1
+; GFX1200-NEXT: v_or_b16 v7.l, v5.l, v7.l
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v8.l, v0.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v2.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
+; GFX1200-NEXT: v_or_b16 v8.h, v10.l, v8.h
+; GFX1200-NEXT: v_or_b16 v4.h, v7.l, v4.h
; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v6.l
-; GFX1200-NEXT: v_and_b16 v4.l, v10.h, 1
-; GFX1200-NEXT: v_mov_b16_e32 v6.l, v11.l
+; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v1.l, v1.l
+; GFX1200-NEXT: v_or_b16 v9.h, v11.l, v9.h
+; GFX1200-NEXT: v_and_b16 v5.h, v5.h, v8.h
+; GFX1200-NEXT: v_lshlrev_b16 v3.l, 8, v3.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v4.h, vcc_lo
-; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v1.h
-; GFX1200-NEXT: v_sub_nc_u16 v4.h, 2, v7.h
-; GFX1200-NEXT: v_or_b16 v4.l, v6.l, v4.l
-; GFX1200-NEXT: v_lshrrev_b16 v6.l, v6.h, v9.h
+; GFX1200-NEXT: v_cndmask_b16 v2.l, v4.h, v2.l, vcc_lo
+; GFX1200-NEXT: v_and_b16 v6.l, v6.h, v9.h
+; GFX1200-NEXT: v_sub_nc_u16 v6.h, 2, v8.l
+; GFX1200-NEXT: v_add_nc_u16 v2.h, v7.h, v5.h
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, v6.l, s0
+; GFX1200-NEXT: v_min_u16 v4.h, v6.h, 15
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v2.h
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v0.l
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v1.l
+; GFX1200-NEXT: v_add_nc_u16 v5.h, v9.l, v5.h
+; GFX1200-NEXT: v_sub_nc_u16 v6.h, v4.h, 1 clamp
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, v2.h, v10.l, vcc_lo
-; GFX1200-NEXT: v_min_u16 v2.h, v4.h, 15
-; GFX1200-NEXT: v_and_b16 v4.h, v5.h, 63
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.l
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.l, v0.h
-; GFX1200-NEXT: v_and_b16 v4.l, v6.l, v4.l
-; GFX1200-NEXT: v_sub_nc_u16 v6.l, v2.h, 1 clamp
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v4.h
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, 7, v7.l
-; GFX1200-NEXT: v_and_b16 v9.l, v5.l, 63
-; GFX1200-NEXT: v_and_b16 v4.h, v5.l, 0x3ff
-; GFX1200-NEXT: v_lshlrev_b16 v7.l, v6.l, 1
+; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
+; GFX1200-NEXT: v_and_b16 v7.h, v6.l, 0x3ff
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s1
-; GFX1200-NEXT: v_and_b16 v8.h, v6.h, 1
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v9.l
-; GFX1200-NEXT: v_or_b16 v9.h, v4.h, 0x400
-; GFX1200-NEXT: v_add_nc_u16 v7.l, v7.l, -1
-; GFX1200-NEXT: v_mov_b16_e32 v10.l, v11.l
-; GFX1200-NEXT: v_lshrrev_b16 v4.h, 7, v4.h
+; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v5.l, s0
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v5.h
+; GFX1200-NEXT: v_lshlrev_b16 v8.h, v6.h, 1
+; GFX1200-NEXT: v_add_nc_u16 v4.l, v4.l, v7.l
+; GFX1200-NEXT: v_or_b16 v5.l, v7.h, 0x400
+; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v1
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, vcc_lo
+; GFX1200-NEXT: v_add_nc_u16 v7.l, v8.h, -1
+; GFX1200-NEXT: v_add_nc_u16 v1.l, v4.l, 6
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v4.l, v5.h, 0, s0
+; GFX1200-NEXT: v_lshlrev_b16 v8.h, 7, v9.l
+; GFX1200-NEXT: v_lshrrev_b16 v9.h, v4.h, v5.l
+; GFX1200-NEXT: v_and_b16 v5.h, v5.l, v7.l
+; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, 8, s0
+; GFX1200-NEXT: v_lshlrev_b16 v9.l, 3, v1.l
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v1.l
+; GFX1200-NEXT: v_lshrrev_b16 v5.l, v6.h, v5.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.h
+; GFX1200-NEXT: v_or_b16 v5.h, v8.h, v7.l
+; GFX1200-NEXT: v_or_b16 v7.l, v8.h, v9.l
+; GFX1200-NEXT: v_and_b16 v9.l, v9.h, 1
+; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v2.l, s1
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s1
-; GFX1200-NEXT: v_lshrrev_b16 v5.h, 6, v5.h
-; GFX1200-NEXT: v_and_b16 v7.l, v9.h, v7.l
-; GFX1200-NEXT: v_or_b16 v8.h, v10.l, v8.h
-; GFX1200-NEXT: v_and_b16 v9.l, v4.h, 1
-; GFX1200-NEXT: v_mov_b16_e32 v10.l, v11.l
-; GFX1200-NEXT: v_lshrrev_b16 v5.l, 6, v5.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v7.l
-; GFX1200-NEXT: v_lshrrev_b16 v11.l, v2.h, v9.h
-; GFX1200-NEXT: v_and_b16 v5.h, v5.h, v8.h
-; GFX1200-NEXT: v_or_b16 v7.l, v10.l, v9.l
-; GFX1200-NEXT: v_lshrrev_b16 v6.l, v6.l, v9.h
+; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v10.h, v0.h
+; GFX1200-NEXT: v_or_b16 v2.h, v7.l, v2.h
+; GFX1200-NEXT: v_or_b16 v5.h, v5.h, v4.l
+; GFX1200-NEXT: v_or_b16 v4.l, v3.h, v3.l
+; GFX1200-NEXT: v_or_b16 v1.l, v10.l, v9.l
+; GFX1200-NEXT: v_sub_nc_u16 v7.l, 2, v10.h
+; GFX1200-NEXT: v_and_b16 v3.h, v6.l, 63
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v5.h, vcc_lo
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v4.h
+; GFX1200-NEXT: v_and_b16 v1.l, v5.l, v1.l
+; GFX1200-NEXT: v_min_u16 v3.l, v7.l, 15
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v4.h, v0.h
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, 7, v7.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v3.h
+; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l
+; GFX1200-NEXT: v_sub_nc_u16 v5.l, v3.l, 1 clamp
+; GFX1200-NEXT: v_and_b16 v3.h, v4.h, 0x3ff
+; GFX1200-NEXT: v_and_b16 v7.h, v4.h, 63
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s1
-; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v1.h, v1.h
+; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0
+; GFX1200-NEXT: v_and_b16 v7.l, v5.h, 1
+; GFX1200-NEXT: v_lshlrev_b16 v6.h, v5.l, 1
+; GFX1200-NEXT: v_or_b16 v10.l, v3.h, 0x400
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.h
+; GFX1200-NEXT: v_mov_b16_e32 v9.l, v11.l
+; GFX1200-NEXT: v_lshrrev_b16 v3.h, 7, v3.h
+; GFX1200-NEXT: v_add_nc_u16 v6.h, v6.h, -1
+; GFX1200-NEXT: v_lshrrev_b16 v4.h, 6, v4.h
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0
+; GFX1200-NEXT: v_or_b16 v7.l, v9.l, v7.l
+; GFX1200-NEXT: v_and_b16 v7.h, v3.h, 1
+; GFX1200-NEXT: v_and_b16 v6.h, v10.l, v6.h
+; GFX1200-NEXT: v_lshrrev_b16 v5.l, v5.l, v10.l
+; GFX1200-NEXT: v_mov_b16_e32 v9.l, v11.l
+; GFX1200-NEXT: v_lshrrev_b16 v11.l, v3.l, v10.l
+; GFX1200-NEXT: v_and_b16 v6.l, v6.l, v7.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.h, 0, v4.l, vcc_lo
-; GFX1200-NEXT: v_and_b16 v4.l, v5.l, v7.l
-; GFX1200-NEXT: v_and_b16 v8.h, v11.l, 1
+; GFX1200-NEXT: v_cndmask_b16 v1.l, 0, v1.l, vcc_lo
+; GFX1200-NEXT: v_or_b16 v6.h, v9.l, v7.h
+; GFX1200-NEXT: v_and_b16 v7.l, v11.l, 1
+; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, v6.l
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v3.l
+; GFX1200-NEXT: v_and_b16 v4.h, v4.h, v6.h
+; GFX1200-NEXT: v_add_nc_u16 v1.l, v9.h, v1.l
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.h
; GFX1200-NEXT: v_mov_b16_e32 v9.l, v12.l
-; GFX1200-NEXT: v_add_nc_u16 v5.l, v6.h, v5.h
-; GFX1200-NEXT: v_add_nc_u16 v1.h, v10.h, v1.h
-; GFX1200-NEXT: v_add_nc_u16 v4.l, v4.h, v4.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v2.h
-; GFX1200-NEXT: v_or_b16 v5.h, v9.l, v8.h
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 7, v1.h
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s4, 7, v4.l
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7f, v1.l, s1
-; GFX1200-NEXT: v_and_b16 v4.h, v6.l, v5.h
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v1.h
+; GFX1200-NEXT: v_add_nc_u16 v3.h, v3.h, v4.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 7, v1.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, 0, s3
-; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s4
-; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, vcc_lo
-; GFX1200-NEXT: v_cndmask_b16 v2.h, 0, v4.h, s2
-; GFX1200-NEXT: v_mov_b16_e32 v6.l, v9.l
-; GFX1200-NEXT: v_lshrrev_b16 v4.h, 8, v0.l
-; GFX1200-NEXT: v_mov_b16_e32 v7.l, v10.l
-; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, 0, s4
-; GFX1200-NEXT: v_add_nc_u16 v2.h, v11.l, v2.h
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v8.l, v6.l
-; GFX1200-NEXT: v_lshrrev_b32_e32 v8, 31, v0
-; GFX1200-NEXT: v_add_nc_u16 v6.h, v7.h, v7.l
-; GFX1200-NEXT: v_cndmask_b16 v6.l, 0, 8, s3
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 7, v2.h
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, 6
-; GFX1200-NEXT: v_and_b16 v4.h, v4.h, 0x80
-; GFX1200-NEXT: v_add_nc_u16 v6.h, v6.h, 6
-; GFX1200-NEXT: v_lshlrev_b16 v8.l, 7, v8.l
+; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, vcc_lo
+; GFX1200-NEXT: v_or_b16 v6.l, v9.l, v7.l
+; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 7, v3.h
+; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v0
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, 8, s2
-; GFX1200-NEXT: v_lshlrev_b16 v7.l, 3, v5.h
-; GFX1200-NEXT: v_or_b16 v6.l, v4.h, v6.l
-; GFX1200-NEXT: v_lshlrev_b16 v8.h, 3, v6.h
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, s2
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v8.h, s0
+; GFX1200-NEXT: v_and_b16 v4.h, v5.l, v6.l
+; GFX1200-NEXT: v_mov_b16_e32 v3.l, v7.l
+; GFX1200-NEXT: v_cndmask_b32_e64 v6, 0, 1, s3
+; GFX1200-NEXT: v_lshrrev_b16 v5.l, 8, v0.l
+; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 8, s2
+; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v4.h, s1
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v8.l, v3.l
+; GFX1200-NEXT: v_add_nc_u16 v6.l, v10.h, v6.l
+; GFX1200-NEXT: v_lshlrev_b16 v8.l, 7, v9.l
+; GFX1200-NEXT: v_and_b16 v5.l, v5.l, 0x80
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v11.l, v4.h
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, 6
+; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, 6
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s3
+; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v1.h, v1.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v4.h
+; GFX1200-NEXT: v_lshlrev_b16 v7.l, 3, v3.l
+; GFX1200-NEXT: v_lshlrev_b16 v9.l, 3, v6.l
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, 0, s2
+; GFX1200-NEXT: v_or_b16 v6.h, v5.l, v6.h
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, 8, s1
+; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s1
+; GFX1200-NEXT: v_or_b16 v9.l, v8.l, v9.l
+; GFX1200-NEXT: v_or_b16 v7.l, v5.l, v7.l
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v3.l
; GFX1200-NEXT: v_or_b16 v7.h, v8.l, v7.h
-; GFX1200-NEXT: v_or_b16 v7.l, v4.h, v7.l
-; GFX1200-NEXT: v_or_b16 v1.h, v6.l, v1.h
-; GFX1200-NEXT: v_or_b16 v8.h, v8.l, v8.h
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v5.h
-; GFX1200-NEXT: v_or_b16 v2.h, v7.h, v2.h
-; GFX1200-NEXT: v_or_b16 v5.l, v7.l, v5.l
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s2, 1, v6.h
-; GFX1200-NEXT: v_or_b16 v4.l, v8.h, v4.l
-; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v6.l
+; GFX1200-NEXT: v_or_b16 v3.h, v9.l, v3.h
+; GFX1200-NEXT: v_cndmask_b16 v2.h, 0x7f, v2.h, s0
+; GFX1200-NEXT: v_or_b16 v1.l, v6.h, v1.l
+; GFX1200-NEXT: v_or_b16 v3.l, v7.h, v4.h
+; GFX1200-NEXT: v_or_b16 v5.h, v7.l, v5.h
+; GFX1200-NEXT: v_and_b16 v2.l, v2.l, 0xff
+; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v0.h, v0.h
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v1.h, v3.h, v3.l, s1
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v0.h
+; GFX1200-NEXT: v_lshlrev_b16 v0.h, 8, v2.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.h, v5.l, v1.h, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v5.h, v1.l, vcc_lo
; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v0.l
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v4.l, v2.h, s2
-; GFX1200-NEXT: v_or_b16 v4.l, v2.l, v3.l
+; GFX1200-NEXT: v_cvt_u32_u16_e32 v3, v4.l
+; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v0.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, v4.h, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, vcc_lo
; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v0.l
-; GFX1200-NEXT: v_cndmask_b16 v0.l, v2.h, v8.l, s0
-; GFX1200-NEXT: v_or_b16 v2.h, v3.h, v1.l
-; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v0.h, v0.h
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v0.h, 0x7f, v1.h, vcc_lo
-; GFX1200-NEXT: v_cvt_u32_u16_e32 v1, v4.l
-; GFX1200-NEXT: v_mov_b16_e32 v3.l, v2.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v0.l, v1.h, v8.l, s1
+; GFX1200-NEXT: v_or_b32_e32 v6, 0x7f7f0000, v3
+; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX1200-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7f, v1.l, vcc_lo
; GFX1200-NEXT: v_cndmask_b16 v0.l, 0x7f, v0.l, s0
-; GFX1200-NEXT: v_and_b16 v0.h, v0.h, 0xff
-; GFX1200-NEXT: v_or_b32_e32 v6, 0x7f7f0000, v1
-; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_lshlrev_b16 v2.l, 8, v0.l
; GFX1200-NEXT: v_lshrrev_b64 v[7:8], 24, v[5:6]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1200-NEXT: v_or_b16 v0.l, v0.h, v2.l
-; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v2
; GFX1200-NEXT: v_lshrrev_b32_e32 v5, 8, v6
-; GFX1200-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_and_b16 v0.h, v1.l, 0xff
+; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v0.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_mov_b16_e32 v3.l, v7.l
+; GFX1200-NEXT: v_or_b16 v0.l, v0.h, v1.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-TRUE16-LABEL: to_fp8_v6f16:
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v3.h, v1
+; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v3.l, v1
; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v7.h, s0
; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v7.l, v2
-; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v3.l, v0
+; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v0.l, v0
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.h
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v7
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.l, v7.l
-; GFX1250-TRUE16-NEXT: v_dual_lshlrev_b32 v6, 16, v1 :: v_dual_lshrrev_b32 v1, 8, v3
+; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
; GFX1250-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[6:7]
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
@@ -2282,498 +2258,503 @@ define <7 x i8> @to_fp8_v7f16(<7 x half> %x) {
; GFX1200-NEXT: s_wait_kmcnt 0x0
; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v4.l, v3.l
; GFX1200-NEXT: v_frexp_mant_f16_e32 v4.h, v3.l
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v6.h, v3.h
; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.h, v3.h
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v6.h, v3.h
; GFX1200-NEXT: v_lshrrev_b16 v5.l, 8, v3.l
; GFX1200-NEXT: v_sub_nc_u16 v6.l, 2, v4.l
-; GFX1200-NEXT: v_and_b16 v8.l, v4.h, 63
; GFX1200-NEXT: v_and_b16 v7.h, v4.h, 0x3ff
-; GFX1200-NEXT: v_sub_nc_u16 v9.l, 2, v6.h
+; GFX1200-NEXT: v_and_b16 v8.l, v4.h, 63
; GFX1200-NEXT: v_and_b16 v10.l, v5.h, 63
+; GFX1200-NEXT: v_sub_nc_u16 v9.l, 2, v6.h
; GFX1200-NEXT: v_min_u16 v6.l, v6.l, 15
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v8.l
-; GFX1200-NEXT: v_or_b16 v10.h, v7.h, 0x400
+; GFX1200-NEXT: v_or_b16 v11.l, v7.h, 0x400
; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v7.h
-; GFX1200-NEXT: v_min_u16 v9.l, v9.l, 15
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v8.l
+; GFX1200-NEXT: v_lshrrev_b16 v4.h, 6, v4.h
; GFX1200-NEXT: v_sub_nc_u16 v9.h, v6.l, 1 clamp
+; GFX1200-NEXT: v_min_u16 v9.l, v9.l, 15
+; GFX1200-NEXT: v_and_b16 v12.l, v7.h, 1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc_lo
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v10.l
-; GFX1200-NEXT: v_and_b16 v10.l, v7.h, 1
-; GFX1200-NEXT: v_sub_nc_u16 v12.h, v9.l, 1 clamp
-; GFX1200-NEXT: v_lshlrev_b16 v11.l, v9.h, 1
-; GFX1200-NEXT: v_lshrrev_b16 v4.h, 6, v4.h
+; GFX1200-NEXT: v_lshlrev_b16 v11.h, v9.h, 1
; GFX1200-NEXT: v_and_b16 v8.h, v5.h, 0x3ff
+; GFX1200-NEXT: v_lshrrev_b16 v9.h, v9.h, v11.l
+; GFX1200-NEXT: v_or_b16 v10.l, v13.l, v12.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc_lo
-; GFX1200-NEXT: v_lshrrev_b16 v11.h, v6.l, v10.h
-; GFX1200-NEXT: v_add_nc_u16 v11.l, v11.l, -1
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v6.l
-; GFX1200-NEXT: v_or_b16 v6.l, v12.l, v10.l
-; GFX1200-NEXT: v_lshlrev_b16 v10.l, v12.h, 1
+; GFX1200-NEXT: v_add_nc_u16 v11.h, v11.h, -1
+; GFX1200-NEXT: v_lshrrev_b16 v12.l, v6.l, v11.l
; GFX1200-NEXT: v_or_b16 v8.l, v8.h, 0x400
-; GFX1200-NEXT: v_and_b16 v11.l, v10.h, v11.l
+; GFX1200-NEXT: v_and_b16 v4.h, v4.h, v10.l
; GFX1200-NEXT: v_lshrrev_b16 v8.h, 7, v8.h
-; GFX1200-NEXT: v_and_b16 v4.h, v4.h, v6.l
-; GFX1200-NEXT: v_add_nc_u16 v10.l, v10.l, -1
-; GFX1200-NEXT: v_and_b16 v6.l, v11.h, 1
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v11.l
-; GFX1200-NEXT: v_lshrrev_b16 v5.h, 6, v5.h
+; GFX1200-NEXT: v_and_b16 v11.h, v11.l, v11.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v9.l
+; GFX1200-NEXT: v_mov_b16_e32 v11.l, v14.l
; GFX1200-NEXT: v_add_nc_u16 v4.h, v7.h, v4.h
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, v9.h, v10.h
-; GFX1200-NEXT: v_and_b16 v9.h, v8.l, v10.l
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
-; GFX1200-NEXT: v_mov_b16_e32 v10.l, v14.l
+; GFX1200-NEXT: v_lshrrev_b16 v7.h, v9.l, v8.l
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v11.h
+; GFX1200-NEXT: v_sub_nc_u16 v11.h, v9.l, 1 clamp
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, 6, v5.h
; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.h
-; GFX1200-NEXT: v_lshrrev_b16 v11.l, v9.l, v8.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v9.h
-; GFX1200-NEXT: v_mov_b16_e32 v12.l, v13.l
-; GFX1200-NEXT: v_and_b16 v13.l, v8.h, 1
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, s0
-; GFX1200-NEXT: v_and_b16 v9.h, v11.l, 1
-; GFX1200-NEXT: v_cndmask_b32_e64 v15, 0, 1, s1
-; GFX1200-NEXT: v_or_b16 v6.l, v12.l, v6.l
; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v7.l, v2.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v6.l
+; GFX1200-NEXT: v_and_b16 v6.l, v12.l, 1
+; GFX1200-NEXT: v_lshlrev_b16 v12.h, v11.h, 1
; GFX1200-NEXT: v_and_b16 v5.l, v5.l, 0x80
+; GFX1200-NEXT: v_mov_b16_e32 v10.l, v13.l
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
+; GFX1200-NEXT: v_sub_nc_u16 v10.h, 2, v7.l
; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s0
-; GFX1200-NEXT: v_mov_b16_e32 v12.l, v15.l
-; GFX1200-NEXT: v_and_b16 v6.l, v7.h, v6.l
-; GFX1200-NEXT: v_or_b16 v7.h, v10.l, v13.l
-; GFX1200-NEXT: v_mov_b16_e32 v10.l, v14.l
+; GFX1200-NEXT: v_cmp_o_f16_e64 s2, v1.l, v1.l
+; GFX1200-NEXT: v_or_b16 v6.l, v10.l, v6.l
+; GFX1200-NEXT: v_add_nc_u16 v10.l, v12.h, -1
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-NEXT: v_and_b16 v6.l, v9.h, v6.l
+; GFX1200-NEXT: v_and_b16 v9.l, v8.l, v10.l
+; GFX1200-NEXT: v_and_b16 v9.h, v8.h, 1
+; GFX1200-NEXT: v_mov_b16_e32 v10.l, v13.l
+; GFX1200-NEXT: v_lshrrev_b16 v8.l, v11.h, v8.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1200-NEXT: v_cndmask_b16 v6.l, 0, v6.l, vcc_lo
-; GFX1200-NEXT: v_and_b16 v5.h, v5.h, v7.h
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, v12.h, v8.l
-; GFX1200-NEXT: v_add_nc_u16 v4.l, v4.l, v10.l
-; GFX1200-NEXT: v_or_b16 v8.l, v12.l, v9.h
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v11.h, v6.l
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v8.h, v5.h
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v9.l
+; GFX1200-NEXT: v_or_b16 v9.l, v11.l, v9.h
+; GFX1200-NEXT: v_add_nc_u16 v4.l, v4.l, v10.l
+; GFX1200-NEXT: v_and_b16 v9.h, v7.h, 1
+; GFX1200-NEXT: v_add_nc_u16 v6.l, v12.l, v6.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc_lo
+; GFX1200-NEXT: v_and_b16 v5.h, v5.h, v9.l
; GFX1200-NEXT: v_add_nc_u16 v4.l, v4.l, 6
-; GFX1200-NEXT: v_and_b16 v7.h, v7.h, v8.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v6.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 7, v5.h
-; GFX1200-NEXT: v_sub_nc_u16 v9.l, 2, v7.l
-; GFX1200-NEXT: v_lshlrev_b16 v8.h, 3, v4.l
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v6.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_mov_b16_e32 v10.l, v13.l
+; GFX1200-NEXT: v_add_nc_u16 v5.h, v8.h, v5.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v4.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, v7.h, vcc_lo
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 8, s1
-; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s2
-; GFX1200-NEXT: v_cndmask_b16 v6.l, v6.l, 0, s1
+; GFX1200-NEXT: v_cndmask_b16 v8.h, 0, 8, vcc_lo
+; GFX1200-NEXT: v_or_b16 v9.l, v10.l, v9.h
+; GFX1200-NEXT: v_lshlrev_b16 v9.h, 3, v4.l
+; GFX1200-NEXT: v_cndmask_b16 v6.l, v6.l, 0, vcc_lo
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.h
; GFX1200-NEXT: v_or_b16 v8.h, v5.l, v8.h
-; GFX1200-NEXT: v_add_nc_u16 v7.h, v11.l, v7.h
-; GFX1200-NEXT: v_or_b16 v8.l, v5.l, v8.l
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.l
-; GFX1200-NEXT: v_min_u16 v4.l, v9.l, 15
-; GFX1200-NEXT: v_add_nc_u16 v6.h, v6.h, v10.l
-; GFX1200-NEXT: v_or_b16 v4.h, v8.h, v4.h
-; GFX1200-NEXT: v_or_b16 v6.l, v8.l, v6.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v7.h
+; GFX1200-NEXT: v_and_b16 v8.l, v8.l, v9.l
+; GFX1200-NEXT: v_or_b16 v9.l, v5.l, v9.h
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, v8.l, s1
+; GFX1200-NEXT: v_or_b16 v4.l, v9.l, v4.h
+; GFX1200-NEXT: v_or_b16 v4.h, v8.h, v6.l
+; GFX1200-NEXT: v_min_u16 v6.l, v10.h, 15
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v8.h, v2.l
+; GFX1200-NEXT: v_add_nc_u16 v7.h, v7.h, v8.l
+; GFX1200-NEXT: v_mov_b16_e32 v8.l, v11.l
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, v4.h, s0
+; GFX1200-NEXT: v_sub_nc_u16 v4.h, v6.l, 1 clamp
; GFX1200-NEXT: v_lshrrev_b32_e32 v10, 31, v3
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v8.l, v2.l
-; GFX1200-NEXT: v_sub_nc_u16 v8.h, v4.l, 1 clamp
-; GFX1200-NEXT: v_add_nc_u16 v6.h, v6.h, 6
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v7.h
+; GFX1200-NEXT: v_add_nc_u16 v6.h, v6.h, v8.l
+; GFX1200-NEXT: v_and_b16 v8.l, v8.h, 0x3ff
+; GFX1200-NEXT: v_lshlrev_b16 v9.l, v4.h, 1
+; GFX1200-NEXT: v_lshlrev_b16 v10.l, 7, v10.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v9.l, 0, 8, s0
-; GFX1200-NEXT: v_lshlrev_b16 v9.h, 7, v10.l
-; GFX1200-NEXT: v_and_b16 v10.l, v8.l, 0x3ff
-; GFX1200-NEXT: v_lshlrev_b16 v10.h, v8.h, 1
-; GFX1200-NEXT: v_lshlrev_b16 v11.l, 3, v6.h
+; GFX1200-NEXT: v_cndmask_b16 v9.h, 0, 8, s0
+; GFX1200-NEXT: v_add_nc_u16 v6.h, v6.h, 6
+; GFX1200-NEXT: v_or_b16 v10.h, v8.l, 0x400
+; GFX1200-NEXT: v_add_nc_u16 v9.l, v9.l, -1
+; GFX1200-NEXT: v_and_b16 v11.h, v8.h, 63
; GFX1200-NEXT: v_cndmask_b16 v7.h, v7.h, 0, s0
-; GFX1200-NEXT: v_or_b16 v9.l, v9.h, v9.l
-; GFX1200-NEXT: v_or_b16 v11.h, v10.l, 0x400
-; GFX1200-NEXT: v_add_nc_u16 v10.h, v10.h, -1
-; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, s2
-; GFX1200-NEXT: v_or_b16 v11.l, v9.h, v11.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, v6.l, vcc_lo
-; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v3.l
-; GFX1200-NEXT: v_and_b16 v6.l, v11.h, v10.h
-; GFX1200-NEXT: v_or_b16 v7.h, v9.l, v7.h
+; GFX1200-NEXT: v_lshlrev_b16 v11.l, 3, v6.h
+; GFX1200-NEXT: v_lshrrev_b16 v12.l, v6.l, v10.h
+; GFX1200-NEXT: v_and_b16 v9.l, v10.h, v9.l
+; GFX1200-NEXT: v_or_b16 v9.h, v10.l, v9.h
+; GFX1200-NEXT: v_lshrrev_b16 v8.l, 7, v8.l
+; GFX1200-NEXT: v_or_b16 v11.l, v10.l, v11.l
+; GFX1200-NEXT: v_lshrrev_b16 v4.h, v4.h, v10.h
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v9.l
+; GFX1200-NEXT: v_and_b16 v9.l, v12.l, 1
+; GFX1200-NEXT: v_or_b16 v7.h, v9.h, v7.h
+; GFX1200-NEXT: v_and_b16 v9.h, v8.l, 1
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v3.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v11.h
; GFX1200-NEXT: v_or_b16 v5.h, v11.l, v5.h
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v6.h
+; GFX1200-NEXT: v_lshrrev_b16 v8.h, 6, v8.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.l
+; GFX1200-NEXT: v_or_b16 v9.l, v13.l, v9.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, v5.l, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v6.l
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, v4.l, v11.h
-; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v3.h
+; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v6.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.h, v7.h, s0
-; GFX1200-NEXT: v_and_b16 v5.h, v8.l, 63
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s1
-; GFX1200-NEXT: v_and_b16 v6.l, v6.h, 1
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v10.l
+; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, v5.l, s1
+; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v3.l, v3.l
+; GFX1200-NEXT: v_and_b16 v4.h, v4.h, v9.l
+; GFX1200-NEXT: v_or_b16 v6.h, v14.l, v9.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, v9.h, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.h
-; GFX1200-NEXT: v_mov_b16_e32 v9.l, v12.l
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v4.l
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v4.l, v2.h
-; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v3.h, v3.h
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0
-; GFX1200-NEXT: v_or_b16 v5.h, v9.l, v6.l
-; GFX1200-NEXT: v_lshrrev_b16 v6.l, v8.h, v11.h
-; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v3.l, v3.l
-; GFX1200-NEXT: v_and_b16 v3.l, v7.h, 1
-; GFX1200-NEXT: v_mov_b16_e32 v9.l, v10.l
-; GFX1200-NEXT: v_lshrrev_b16 v8.h, 8, v2.l
-; GFX1200-NEXT: v_and_b16 v5.h, v6.l, v5.h
-; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v8.l
+; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.h, v7.h, vcc_lo
+; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v3.h
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v7.h, v2.h
+; GFX1200-NEXT: v_cndmask_b16 v3.l, 0, v4.h, s0
+; GFX1200-NEXT: v_and_b16 v5.h, v8.h, v6.h
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v4.h, v2.h
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, v10.l, vcc_lo
+; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v3.h, v3.h
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v12.l, v3.l
+; GFX1200-NEXT: v_add_nc_u16 v5.h, v8.l, v5.h
+; GFX1200-NEXT: v_sub_nc_u16 v3.h, 2, v4.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.h, 0x7f, v4.h, s0
-; GFX1200-NEXT: v_or_b16 v3.l, v9.l, v3.l
-; GFX1200-NEXT: v_sub_nc_u16 v4.h, 2, v4.l
+; GFX1200-NEXT: v_cndmask_b16 v4.l, 0x7f, v4.l, s1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, v5.h, vcc_lo
-; GFX1200-NEXT: v_and_b16 v8.h, v8.h, 0x80
-; GFX1200-NEXT: v_cndmask_b16 v5.l, 0x7f, v5.l, s1
-; GFX1200-NEXT: v_and_b16 v3.l, v6.l, v3.l
-; GFX1200-NEXT: v_min_u16 v4.h, v4.h, 15
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v6.h, v5.h
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v2.h
+; GFX1200-NEXT: v_cndmask_b16 v5.l, 0x7f, v5.l, vcc_lo
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v3.l
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v5.h
+; GFX1200-NEXT: v_min_u16 v3.h, v3.h, 15
; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v2.l, v2.l
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v7.h, v3.l
-; GFX1200-NEXT: v_sub_nc_u16 v6.h, v4.h, 1 clamp
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.h
-; GFX1200-NEXT: v_and_b16 v7.h, v6.l, 0x3ff
-; GFX1200-NEXT: v_and_b16 v10.h, v6.l, 63
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v3.l
-; GFX1200-NEXT: v_lshlrev_b16 v8.l, v6.h, 1
+; GFX1200-NEXT: v_and_b16 v4.l, v4.l, 0xff
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, vcc_lo
-; GFX1200-NEXT: v_cndmask_b16 v10.l, 0, 8, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v10.h
+; GFX1200-NEXT: v_cndmask_b16 v8.h, 0, 8, vcc_lo
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, s0
-; GFX1200-NEXT: v_or_b16 v9.h, v7.h, 0x400
-; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, -1
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v7.h
+; GFX1200-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0
+; GFX1200-NEXT: v_lshrrev_b16 v6.h, 8, v2.l
+; GFX1200-NEXT: v_sub_nc_u16 v8.l, v3.h, 1 clamp
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, s0
+; GFX1200-NEXT: v_add_nc_u16 v6.l, v7.l, v6.l
+; GFX1200-NEXT: v_and_b16 v6.h, v6.h, 0x80
+; GFX1200-NEXT: v_and_b16 v7.l, v7.h, 0x3ff
+; GFX1200-NEXT: v_lshlrev_b16 v9.l, v8.l, 1
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v3.h
+; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, 6
+; GFX1200-NEXT: v_or_b16 v8.h, v6.h, v8.h
+; GFX1200-NEXT: v_or_b16 v9.h, v7.l, 0x400
+; GFX1200-NEXT: v_add_nc_u16 v9.l, v9.l, -1
+; GFX1200-NEXT: v_lshrrev_b16 v7.l, 7, v7.l
+; GFX1200-NEXT: v_lshlrev_b16 v10.l, 3, v6.l
+; GFX1200-NEXT: v_or_b16 v3.l, v8.h, v3.l
+; GFX1200-NEXT: v_lshrrev_b16 v10.h, v3.h, v9.h
+; GFX1200-NEXT: v_and_b16 v8.h, v9.h, v9.l
+; GFX1200-NEXT: v_lshrrev_b16 v8.l, v8.l, v9.h
+; GFX1200-NEXT: v_or_b16 v9.l, v6.h, v10.l
+; GFX1200-NEXT: v_and_b16 v10.l, v7.h, 63
+; GFX1200-NEXT: v_lshrrev_b16 v7.h, 6, v7.h
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v8.h
+; GFX1200-NEXT: v_and_b16 v8.h, v10.h, 1
+; GFX1200-NEXT: v_or_b16 v5.h, v9.l, v5.h
+; GFX1200-NEXT: v_and_b16 v9.l, v7.l, 1
+; GFX1200-NEXT: v_lshlrev_b16 v5.l, 8, v5.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
-; GFX1200-NEXT: v_add_nc_u16 v7.l, v7.l, v9.l
-; GFX1200-NEXT: v_lshrrev_b16 v9.l, v4.h, v9.h
-; GFX1200-NEXT: v_and_b16 v8.l, v9.h, v8.l
-; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, v6.h, v9.h
-; GFX1200-NEXT: v_add_nc_u16 v7.l, v7.l, 6
-; GFX1200-NEXT: v_and_b16 v10.h, v9.l, 1
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v8.l
-; GFX1200-NEXT: v_and_b16 v8.l, v7.h, 1
-; GFX1200-NEXT: v_or_b16 v10.l, v8.h, v10.l
-; GFX1200-NEXT: v_lshlrev_b16 v11.h, 3, v7.l
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s0
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v10.l
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v11.h, v1.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_mov_b16_e32 v10.l, v11.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo
-; GFX1200-NEXT: v_or_b16 v8.l, v11.l, v8.l
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v9.h, v1.l
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v4.h
-; GFX1200-NEXT: v_or_b16 v5.h, v10.l, v5.h
-; GFX1200-NEXT: v_or_b16 v10.h, v12.l, v10.h
-; GFX1200-NEXT: v_and_b16 v6.l, v6.l, v8.l
-; GFX1200-NEXT: v_or_b16 v8.l, v8.h, v11.h
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v7.l
-; GFX1200-NEXT: v_and_b16 v3.h, v3.h, 0xff
-; GFX1200-NEXT: v_and_b16 v6.h, v6.h, v10.h
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v7.h, v6.l
-; GFX1200-NEXT: v_or_b16 v3.l, v8.l, v3.l
-; GFX1200-NEXT: v_sub_nc_u16 v6.l, 2, v9.h
-; GFX1200-NEXT: v_lshrrev_b32_e32 v10, 31, v2
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v6.l
+; GFX1200-NEXT: v_or_b16 v8.h, v10.l, v8.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1200-NEXT: v_mov_b16_e32 v11.l, v12.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, v6.h, vcc_lo
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v4.h
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v5.h, s0
-; GFX1200-NEXT: v_min_u16 v5.h, v6.l, 15
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v2.l
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v9.l, v6.h
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v5.h, v3.l, vcc_lo
+; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v2.l
+; GFX1200-NEXT: v_and_b16 v8.l, v8.l, v8.h
+; GFX1200-NEXT: v_or_b16 v6.l, v11.l, v9.l
+; GFX1200-NEXT: v_sub_nc_u16 v9.l, 2, v11.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.h, v1.l
-; GFX1200-NEXT: v_sub_nc_u16 v7.h, v5.h, 1 clamp
-; GFX1200-NEXT: v_lshlrev_b16 v10.l, 7, v10.l
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v6.h, vcc_lo
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_and_b16 v5.h, v7.h, v6.l
+; GFX1200-NEXT: v_min_u16 v3.h, v9.l, 15
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v8.h, s0
-; GFX1200-NEXT: v_add_nc_u16 v4.l, v4.l, v7.l
-; GFX1200-NEXT: v_and_b16 v7.l, v6.h, 0x3ff
-; GFX1200-NEXT: v_lshlrev_b16 v8.l, v7.h, 1
+; GFX1200-NEXT: v_cndmask_b16 v6.l, 0, v8.l, s0
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v7.h, v1.l
+; GFX1200-NEXT: v_cndmask_b16 v3.l, 0x7f, v3.l, s1
+; GFX1200-NEXT: v_add_nc_u16 v5.h, v7.l, v5.h
+; GFX1200-NEXT: v_sub_nc_u16 v8.l, v3.h, 1 clamp
+; GFX1200-NEXT: v_add_nc_u16 v6.l, v10.h, v6.l
+; GFX1200-NEXT: v_and_b16 v6.h, v7.h, 0x3ff
+; GFX1200-NEXT: v_lshrrev_b32_e32 v10, 31, v2
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.h
+; GFX1200-NEXT: v_lshlrev_b16 v7.l, v8.l, 1
; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v6.l
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, vcc_lo
-; GFX1200-NEXT: v_add_nc_u16 v4.l, v4.l, 6
-; GFX1200-NEXT: v_or_b16 v9.l, v7.l, 0x400
-; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, -1
+; GFX1200-NEXT: v_or_b16 v8.h, v6.h, 0x400
+; GFX1200-NEXT: v_lshrrev_b16 v6.h, 7, v6.h
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc_lo
+; GFX1200-NEXT: v_add_nc_u16 v7.l, v7.l, -1
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v8.h, 0, 8, s0
-; GFX1200-NEXT: v_cndmask_b16 v6.l, v6.l, 0, s0
-; GFX1200-NEXT: v_lshlrev_b16 v2.l, 3, v4.l
-; GFX1200-NEXT: v_lshrrev_b16 v10.h, v5.h, v9.l
-; GFX1200-NEXT: v_and_b16 v8.l, v9.l, v8.l
-; GFX1200-NEXT: v_or_b16 v8.h, v10.l, v8.h
-; GFX1200-NEXT: v_cndmask_b16 v3.l, 0x7f, v3.l, s1
-; GFX1200-NEXT: v_or_b16 v2.l, v10.l, v2.l
-; GFX1200-NEXT: v_lshlrev_b16 v5.l, 8, v5.l
+; GFX1200-NEXT: v_cndmask_b16 v2.l, v6.l, 0, s0
+; GFX1200-NEXT: v_lshrrev_b16 v9.h, v3.h, v8.h
+; GFX1200-NEXT: v_lshrrev_b16 v8.l, v8.l, v8.h
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, v9.l
+; GFX1200-NEXT: v_and_b16 v6.l, v8.h, v7.l
+; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, 8, s0
+; GFX1200-NEXT: v_lshlrev_b16 v9.l, 7, v10.l
+; GFX1200-NEXT: v_and_b16 v10.l, v9.h, 1
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, 6
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.l
+; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, vcc_lo
+; GFX1200-NEXT: v_or_b16 v6.l, v9.l, v7.l
+; GFX1200-NEXT: v_and_b16 v7.l, v7.h, 63
+; GFX1200-NEXT: v_lshlrev_b16 v10.h, 3, v4.h
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0
+; GFX1200-NEXT: v_and_b16 v8.h, v6.h, 1
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.l
+; GFX1200-NEXT: v_or_b16 v10.h, v9.l, v10.h
+; GFX1200-NEXT: v_mov_b16_e32 v11.l, v12.l
+; GFX1200-NEXT: v_or_b16 v2.l, v6.l, v2.l
+; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v7.h
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v7.h, v1.h
+; GFX1200-NEXT: v_or_b16 v7.l, v11.l, v10.l
+; GFX1200-NEXT: v_or_b16 v5.h, v10.h, v5.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_mov_b16_e32 v10.l, v12.l
+; GFX1200-NEXT: v_and_b16 v7.l, v8.l, v7.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1200-NEXT: v_or_b16 v3.h, v10.l, v8.h
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, v7.l, vcc_lo
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_and_b16 v3.h, v6.l, v3.h
+; GFX1200-NEXT: v_sub_nc_u16 v6.l, 2, v7.h
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v9.h, v7.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v2.l, v5.h, v2.l, vcc_lo
+; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v2.h
+; GFX1200-NEXT: v_add_nc_u16 v3.h, v6.h, v3.h
+; GFX1200-NEXT: v_min_u16 v5.h, v6.l, 15
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.h
+; GFX1200-NEXT: v_lshrrev_b16 v6.l, 8, v1.l
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.h, v1.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v3.h
+; GFX1200-NEXT: v_sub_nc_u16 v7.l, v5.h, 1 clamp
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 8, s0
+; GFX1200-NEXT: v_and_b16 v6.l, v6.l, 0x80
+; GFX1200-NEXT: v_and_b16 v8.h, v6.h, 0x3ff
+; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s0
+; GFX1200-NEXT: v_lshlrev_b16 v9.h, v7.l, 1
+; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s1
+; GFX1200-NEXT: v_or_b16 v8.l, v6.l, v8.l
+; GFX1200-NEXT: v_or_b16 v10.h, v8.h, 0x400
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v9.l, vcc_lo
+; GFX1200-NEXT: v_add_nc_u16 v9.h, v9.h, -1
+; GFX1200-NEXT: v_add_nc_u16 v10.l, v11.h, v10.l
+; GFX1200-NEXT: v_or_b16 v4.h, v8.l, v4.h
+; GFX1200-NEXT: v_and_b16 v8.l, v6.h, 63
+; GFX1200-NEXT: v_lshrrev_b16 v8.h, 7, v8.h
+; GFX1200-NEXT: v_and_b16 v9.h, v10.h, v9.h
+; GFX1200-NEXT: v_add_nc_u16 v9.l, v10.l, 6
+; GFX1200-NEXT: v_lshrrev_b16 v10.l, v5.h, v10.h
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v8.l
-; GFX1200-NEXT: v_and_b16 v8.l, v10.h, 1
-; GFX1200-NEXT: v_or_b16 v6.l, v8.h, v6.l
-; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v4.h
-; GFX1200-NEXT: v_and_b16 v4.h, v6.h, 63
+; GFX1200-NEXT: v_lshrrev_b16 v6.h, 6, v6.h
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s1
+; GFX1200-NEXT: v_lshlrev_b16 v8.l, 3, v9.l
+; GFX1200-NEXT: v_lshrrev_b16 v7.l, v7.l, v10.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.l
-; GFX1200-NEXT: v_lshrrev_b16 v4.l, 7, v7.l
-; GFX1200-NEXT: v_lshrrev_b16 v7.l, v7.h, v9.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v4.h
-; GFX1200-NEXT: v_or_b16 v4.h, v11.l, v8.l
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v8.l, v1.h
-; GFX1200-NEXT: v_and_b16 v7.h, v4.l, 1
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v9.h
+; GFX1200-NEXT: v_and_b16 v9.h, v8.h, 1
+; GFX1200-NEXT: v_and_b16 v11.h, v10.l, 1
+; GFX1200-NEXT: v_or_b16 v8.l, v6.l, v8.l
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v10.h, v0.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v6.l, vcc_lo
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.h
-; GFX1200-NEXT: v_sub_nc_u16 v5.h, 2, v8.l
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, 6, v6.h
-; GFX1200-NEXT: v_and_b16 v4.h, v7.l, v4.h
-; GFX1200-NEXT: v_mov_b16_e32 v9.l, v12.l
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v2.h
-; GFX1200-NEXT: v_min_u16 v5.h, v5.h, 15
+; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo
+; GFX1200-NEXT: v_or_b16 v9.h, v11.l, v9.h
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v9.l
+; GFX1200-NEXT: v_or_b16 v3.h, v8.l, v3.h
+; GFX1200-NEXT: v_sub_nc_u16 v8.l, 2, v10.h
+; GFX1200-NEXT: v_or_b16 v11.l, v12.l, v11.h
+; GFX1200-NEXT: v_and_b16 v6.h, v6.h, v9.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.h
+; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v2.h, v2.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v4.h, vcc_lo
-; GFX1200-NEXT: v_or_b16 v6.l, v9.l, v7.h
-; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v2.h, v2.h
-; GFX1200-NEXT: v_sub_nc_u16 v7.l, v5.h, 1 clamp
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v3.h, v4.h, vcc_lo
+; GFX1200-NEXT: v_and_b16 v7.l, v7.l, v11.l
+; GFX1200-NEXT: v_add_nc_u16 v5.h, v8.h, v6.h
+; GFX1200-NEXT: v_min_u16 v3.h, v8.l, 15
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.h, v0.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v10.l, s0
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v10.h, v4.h
-; GFX1200-NEXT: v_and_b16 v6.l, v6.h, v6.l
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.h, v1.h
-; GFX1200-NEXT: v_lshrrev_b16 v9.l, 8, v1.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.h
-; GFX1200-NEXT: v_add_nc_u16 v4.l, v4.l, v6.l
+; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v2.l, s1
+; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v7.l, s0
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.h
+; GFX1200-NEXT: v_sub_nc_u16 v7.l, v3.h, 1 clamp
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v1.l
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1200-NEXT: v_and_b16 v2.h, v6.h, 0x3ff
-; GFX1200-NEXT: v_lshlrev_b16 v6.l, v7.l, 1
-; GFX1200-NEXT: v_and_b16 v10.l, v6.h, 63
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, 6, v6.h
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v4.l
-; GFX1200-NEXT: v_or_b16 v7.h, v2.h, 0x400
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, -1
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v10.l
-; GFX1200-NEXT: v_lshrrev_b16 v2.h, 7, v2.h
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v10.l, v4.h
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
+; GFX1200-NEXT: v_and_b16 v8.h, v6.h, 0x3ff
+; GFX1200-NEXT: v_lshlrev_b16 v9.l, v7.l, 1
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s1
-; GFX1200-NEXT: v_lshrrev_b16 v10.h, v5.h, v7.h
-; GFX1200-NEXT: v_and_b16 v6.l, v7.h, v6.l
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s2
-; GFX1200-NEXT: v_lshrrev_b16 v7.l, v7.l, v7.h
-; GFX1200-NEXT: v_cndmask_b16 v8.h, 0, 8, s0
-; GFX1200-NEXT: v_and_b16 v9.l, v9.l, 0x80
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v6.l
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v9.h, v11.l
-; GFX1200-NEXT: v_and_b16 v9.h, v2.h, 1
-; GFX1200-NEXT: v_mov_b16_e32 v10.l, v12.l
-; GFX1200-NEXT: v_and_b16 v11.l, v10.h, 1
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v6.l, s0
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.h
+; GFX1200-NEXT: v_add_nc_u16 v6.l, v7.h, v8.l
+; GFX1200-NEXT: v_or_b16 v7.h, v8.h, 0x400
+; GFX1200-NEXT: v_add_nc_u16 v8.l, v9.l, -1
+; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v1
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s2
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, 6
; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s0
-; GFX1200-NEXT: v_or_b16 v9.h, v10.l, v9.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.h
-; GFX1200-NEXT: v_mov_b16_e32 v12.l, v13.l
-; GFX1200-NEXT: v_or_b16 v7.h, v9.l, v8.h
-; GFX1200-NEXT: v_lshlrev_b16 v8.h, 3, v6.l
-; GFX1200-NEXT: v_and_b16 v6.h, v6.h, v9.h
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v9.h, v0.l
-; GFX1200-NEXT: v_or_b16 v10.l, v12.l, v11.l
-; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, 0, s1
-; GFX1200-NEXT: v_or_b16 v5.h, v9.l, v8.h
-; GFX1200-NEXT: v_add_nc_u16 v2.h, v2.h, v6.h
-; GFX1200-NEXT: v_sub_nc_u16 v6.h, 2, v9.h
-; GFX1200-NEXT: v_and_b16 v7.l, v7.l, v10.l
-; GFX1200-NEXT: v_or_b16 v4.h, v7.h, v4.h
-; GFX1200-NEXT: v_or_b16 v4.l, v5.h, v4.l
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v7.h, v0.l
-; GFX1200-NEXT: v_min_u16 v6.h, v6.h, 15
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, v7.l, s0
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v2.h
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v6.l
-; GFX1200-NEXT: v_sub_nc_u16 v8.h, v6.h, 1 clamp
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v10.h, v7.l
+; GFX1200-NEXT: v_add_nc_u16 v1.l, v6.l, 6
+; GFX1200-NEXT: v_lshrrev_b16 v10.l, v3.h, v7.h
+; GFX1200-NEXT: v_and_b16 v6.l, v7.h, v8.l
+; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 8, s0
+; GFX1200-NEXT: v_lshlrev_b16 v9.l, 7, v9.l
+; GFX1200-NEXT: v_lshlrev_b16 v9.h, 3, v1.l
+; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, vcc_lo
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.l
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v1.l
+; GFX1200-NEXT: v_or_b16 v6.l, v9.l, v8.l
+; GFX1200-NEXT: v_or_b16 v8.l, v9.l, v9.h
+; GFX1200-NEXT: v_and_b16 v9.h, v10.l, 1
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0
-; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, v4.h, s1
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, s0
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v1.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 7, v5.h
-; GFX1200-NEXT: v_mov_b16_e32 v7.l, v11.l
-; GFX1200-NEXT: v_lshrrev_b32_e32 v11, 31, v1
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, v9.l, s1
-; GFX1200-NEXT: v_cndmask_b16 v10.l, 0, 8, s2
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v8.l, v7.l
-; GFX1200-NEXT: v_and_b16 v7.l, v7.h, 0x3ff
-; GFX1200-NEXT: v_lshlrev_b16 v8.l, v8.h, 1
-; GFX1200-NEXT: v_lshlrev_b16 v11.l, 7, v11.l
-; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, s2
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, 6
-; GFX1200-NEXT: v_or_b16 v10.h, v7.l, 0x400
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v11.h, v0.h
+; GFX1200-NEXT: v_or_b16 v4.h, v6.l, v4.h
+; GFX1200-NEXT: v_or_b16 v5.h, v8.l, v5.h
+; GFX1200-NEXT: v_lshrrev_b16 v7.l, v7.l, v7.h
+; GFX1200-NEXT: v_or_b16 v1.l, v11.l, v9.h
+; GFX1200-NEXT: v_sub_nc_u16 v6.l, 2, v11.h
+; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v8.h
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v4.h, v5.h, v4.h, vcc_lo
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.h
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v3.h, v0.h
+; GFX1200-NEXT: v_min_u16 v5.h, v6.l, 15
+; GFX1200-NEXT: v_and_b16 v6.l, v6.h, 63
+; GFX1200-NEXT: v_and_b16 v1.l, v7.l, v1.l
+; GFX1200-NEXT: v_and_b16 v8.h, v7.h, 1
+; GFX1200-NEXT: v_and_b16 v9.h, v3.h, 63
+; GFX1200-NEXT: v_sub_nc_u16 v7.l, v5.h, 1 clamp
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.l
+; GFX1200-NEXT: v_and_b16 v6.l, v3.h, 0x3ff
+; GFX1200-NEXT: v_lshrrev_b16 v6.h, 6, v6.h
+; GFX1200-NEXT: v_lshrrev_b16 v3.h, 6, v3.h
+; GFX1200-NEXT: v_lshlrev_b16 v8.l, v7.l, 1
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v9.h
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v1.l, 0, v1.l, vcc_lo
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v5.h
; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, -1
-; GFX1200-NEXT: v_or_b16 v10.l, v11.l, v10.l
-; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v1.l, v1.l
-; GFX1200-NEXT: v_lshlrev_b16 v4.h, 3, v6.l
-; GFX1200-NEXT: v_lshrrev_b16 v11.h, v6.h, v10.h
-; GFX1200-NEXT: v_and_b16 v8.l, v10.h, v8.l
-; GFX1200-NEXT: v_or_b16 v5.h, v10.l, v5.h
-; GFX1200-NEXT: v_lshrrev_b16 v7.l, 7, v7.l
-; GFX1200-NEXT: v_or_b16 v4.h, v11.l, v4.h
+; GFX1200-NEXT: v_mov_b16_e32 v11.l, v12.l
+; GFX1200-NEXT: v_or_b16 v12.l, v6.l, 0x400
+; GFX1200-NEXT: v_lshrrev_b16 v6.l, 7, v6.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v4.l, 0x7f, v4.l, s1
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
+; GFX1200-NEXT: v_add_nc_u16 v1.l, v10.l, v1.l
+; GFX1200-NEXT: v_or_b16 v8.h, v11.l, v8.h
+; GFX1200-NEXT: v_and_b16 v8.l, v12.l, v8.l
+; GFX1200-NEXT: v_and_b16 v9.h, v6.l, 1
+; GFX1200-NEXT: v_mov_b16_e32 v11.l, v13.l
+; GFX1200-NEXT: v_lshrrev_b16 v12.h, v5.h, v12.l
+; GFX1200-NEXT: v_and_b16 v6.h, v6.h, v8.h
; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v8.l
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v8.l, v0.h
-; GFX1200-NEXT: v_and_b16 v9.l, v7.l, 1
-; GFX1200-NEXT: v_or_b16 v2.h, v4.h, v2.h
-; GFX1200-NEXT: v_and_b16 v4.h, v11.h, 1
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v6.l
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v0.h
-; GFX1200-NEXT: v_and_b16 v4.l, v4.l, 0xff
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1200-NEXT: v_mov_b16_e32 v6.l, v12.l
+; GFX1200-NEXT: v_lshrrev_b16 v7.l, v7.l, v12.l
+; GFX1200-NEXT: v_or_b16 v8.l, v11.l, v9.h
+; GFX1200-NEXT: v_and_b16 v8.h, v12.h, 1
+; GFX1200-NEXT: v_add_nc_u16 v6.h, v7.h, v6.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v5.h, s0
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
+; GFX1200-NEXT: v_cndmask_b16 v2.h, 0x7f, v2.h, s2
+; GFX1200-NEXT: v_and_b16 v3.h, v3.h, v8.l
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 7, v1.l
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v6.h
+; GFX1200-NEXT: v_mov_b16_e32 v11.l, v13.l
; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v1.h
-; GFX1200-NEXT: v_sub_nc_u16 v5.h, 2, v8.l
-; GFX1200-NEXT: v_or_b16 v4.h, v6.l, v4.h
-; GFX1200-NEXT: v_lshrrev_b16 v6.l, v8.h, v10.h
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, v2.h, v11.l, s0
-; GFX1200-NEXT: v_min_u16 v2.h, v5.h, 15
-; GFX1200-NEXT: v_and_b16 v5.h, v7.h, 63
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.h
-; GFX1200-NEXT: v_and_b16 v4.h, v6.l, v4.h
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v0.h
-; GFX1200-NEXT: v_sub_nc_u16 v6.h, v2.h, 1 clamp
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v5.h
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, 6, v7.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s3, 0, v2.h
-; GFX1200-NEXT: v_and_b16 v10.l, v6.l, 63
-; GFX1200-NEXT: v_and_b16 v5.h, v6.l, 0x3ff
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s2
-; GFX1200-NEXT: v_lshlrev_b16 v8.h, v6.h, 1
-; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v10.l
-; GFX1200-NEXT: v_or_b16 v10.h, v5.h, 0x400
-; GFX1200-NEXT: v_mov_b16_e32 v11.l, v12.l
-; GFX1200-NEXT: v_add_nc_u16 v8.h, v8.h, -1
-; GFX1200-NEXT: v_lshrrev_b16 v5.h, 7, v5.h
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s2
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, v6.h, v10.h
-; GFX1200-NEXT: v_or_b16 v9.l, v11.l, v9.l
-; GFX1200-NEXT: v_and_b16 v8.h, v10.h, v8.h
-; GFX1200-NEXT: v_and_b16 v10.l, v5.h, 1
-; GFX1200-NEXT: v_mov_b16_e32 v11.l, v12.l
-; GFX1200-NEXT: v_lshrrev_b16 v12.l, v2.h, v10.h
-; GFX1200-NEXT: v_and_b16 v7.h, v7.h, v9.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v8.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_or_b16 v8.h, v11.l, v10.l
-; GFX1200-NEXT: v_and_b16 v9.l, v12.l, 1
+; GFX1200-NEXT: v_add_nc_u16 v3.h, v6.l, v3.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s2
-; GFX1200-NEXT: v_cmp_o_f16_e64 s2, v1.h, v1.h
-; GFX1200-NEXT: v_cndmask_b16 v1.h, 0, v4.h, s0
-; GFX1200-NEXT: v_and_b16 v4.h, v6.l, v8.h
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v7.l, v7.h
-; GFX1200-NEXT: v_mov_b16_e32 v10.l, v13.l
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, 0, s2
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v6.h, v6.h, 0, vcc_lo
+; GFX1200-NEXT: v_or_b16 v7.h, v11.l, v8.h
+; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 7, v3.h
+; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, v9.l, s0
+; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v1.h, v1.h
+; GFX1200-NEXT: v_and_b16 v6.l, v7.l, v7.h
+; GFX1200-NEXT: v_mov_b16_e32 v7.l, v8.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7f, v1.l, s2
-; GFX1200-NEXT: v_add_nc_u16 v1.h, v11.h, v1.h
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v5.h, v4.h
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v6.l
-; GFX1200-NEXT: v_or_b16 v7.l, v10.l, v9.l
-; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s4, 7, v1.h
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s5, 7, v4.h
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s3
+; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, 8, s2
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s3
+; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, v6.l, s1
+; GFX1200-NEXT: v_add_nc_u16 v7.l, v10.h, v7.l
+; GFX1200-NEXT: v_mov_b16_e32 v8.l, v13.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v10, 31, v0
+; GFX1200-NEXT: v_lshrrev_b16 v6.l, 8, v0.l
+; GFX1200-NEXT: v_add_nc_u16 v5.h, v12.h, v5.h
+; GFX1200-NEXT: v_add_nc_u16 v7.l, v7.l, 6
+; GFX1200-NEXT: v_add_nc_u16 v8.l, v11.h, v8.l
+; GFX1200-NEXT: v_lshlrev_b16 v10.l, 7, v10.l
+; GFX1200-NEXT: v_and_b16 v6.l, v6.l, 0x80
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v5.h
+; GFX1200-NEXT: v_lshlrev_b16 v8.h, 3, v7.l
+; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, 6
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v7.l
+; GFX1200-NEXT: v_or_b16 v7.h, v6.l, v7.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0
-; GFX1200-NEXT: v_and_b16 v5.h, v6.h, v7.l
-; GFX1200-NEXT: v_cndmask_b16 v6.l, v6.l, 0, s0
-; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, 0, s4
-; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s5
-; GFX1200-NEXT: v_mov_b16_e32 v7.l, v10.l
-; GFX1200-NEXT: v_cndmask_b16 v2.h, 0, v5.h, s3
-; GFX1200-NEXT: v_lshrrev_b16 v5.h, 8, v0.l
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s5
-; GFX1200-NEXT: v_mov_b16_e32 v9.l, v11.l
-; GFX1200-NEXT: v_add_nc_u16 v6.h, v9.h, v7.l
-; GFX1200-NEXT: v_add_nc_u16 v2.h, v12.l, v2.h
-; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, 8, s4
-; GFX1200-NEXT: v_and_b16 v5.h, v5.h, 0x80
-; GFX1200-NEXT: v_add_nc_u16 v7.h, v8.l, v9.l
-; GFX1200-NEXT: v_add_nc_u16 v6.h, v6.h, 6
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 7, v2.h
-; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v0
-; GFX1200-NEXT: v_or_b16 v7.l, v5.h, v7.l
-; GFX1200-NEXT: v_add_nc_u16 v7.h, v7.h, 6
-; GFX1200-NEXT: v_lshlrev_b16 v8.l, 3, v6.h
+; GFX1200-NEXT: v_cndmask_b16 v9.h, 0, 8, s1
+; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, s1
+; GFX1200-NEXT: v_lshlrev_b16 v10.h, 3, v8.l
+; GFX1200-NEXT: v_or_b16 v8.h, v6.l, v8.h
+; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v8.l
+; GFX1200-NEXT: v_or_b16 v9.h, v10.l, v9.h
+; GFX1200-NEXT: v_or_b16 v1.l, v7.h, v1.l
+; GFX1200-NEXT: v_or_b16 v10.h, v10.l, v10.h
+; GFX1200-NEXT: v_or_b16 v6.h, v8.h, v6.h
+; GFX1200-NEXT: v_and_b16 v2.h, v2.h, 0xff
+; GFX1200-NEXT: v_or_b16 v5.h, v9.h, v5.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1200-NEXT: v_or_b16 v3.h, v10.h, v3.h
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v6.h, v1.l, vcc_lo
+; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v0.l
+; GFX1200-NEXT: v_or_b16 v6.h, v4.l, v5.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v8.h, 0, 8, s3
-; GFX1200-NEXT: v_lshlrev_b16 v9.l, 7, v9.l
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, s3
-; GFX1200-NEXT: v_lshlrev_b16 v9.h, 3, v7.h
-; GFX1200-NEXT: v_or_b16 v8.l, v5.h, v8.l
-; GFX1200-NEXT: v_or_b16 v1.h, v7.l, v1.h
-; GFX1200-NEXT: v_or_b16 v8.h, v9.l, v8.h
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v6.h
-; GFX1200-NEXT: v_or_b16 v9.h, v9.l, v9.h
-; GFX1200-NEXT: v_or_b16 v6.l, v8.l, v6.l
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s3, 1, v7.h
-; GFX1200-NEXT: v_or_b16 v2.h, v8.h, v2.h
-; GFX1200-NEXT: v_or_b16 v6.h, v3.h, v5.l
-; GFX1200-NEXT: v_or_b16 v4.h, v9.h, v4.h
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.h, v6.l, v1.h, s0
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v0.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v4.h, v2.h, s3
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, v5.h, s0
-; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v0.l, v0.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1200-NEXT: v_cndmask_b16 v0.l, v2.h, v9.l, s1
-; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v0.h, v0.h
+; GFX1200-NEXT: v_cndmask_b16 v1.h, v3.h, v5.h, s1
+; GFX1200-NEXT: v_cndmask_b16 v3.h, 0x7f, v4.h, s0
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v0.h
+; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v0.h, v0.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v0.h, 0x7f, v2.l, vcc_lo
-; GFX1200-NEXT: v_or_b16 v2.h, v4.l, v1.l
-; GFX1200-NEXT: v_and_b16 v1.l, v3.l, 0xff
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v6.l, vcc_lo
+; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v0.l
+; GFX1200-NEXT: v_lshlrev_b16 v0.h, 8, v3.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.h, 0x7f, v1.h, s0
-; GFX1200-NEXT: v_cndmask_b16 v0.l, 0x7f, v0.l, s1
-; GFX1200-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
-; GFX1200-NEXT: v_mov_b16_e32 v4.l, v2.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_and_b16 v1.h, v1.h, 0xff
-; GFX1200-NEXT: v_lshlrev_b16 v2.l, 8, v0.l
+; GFX1200-NEXT: v_cndmask_b16 v0.l, v1.h, v10.l, s1
+; GFX1200-NEXT: v_and_b16 v1.h, v3.l, 0xff
+; GFX1200-NEXT: v_lshlrev_b16 v3.l, 8, v2.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7f, v1.l, vcc_lo
+; GFX1200-NEXT: v_or_b16 v2.l, v2.h, v0.h
+; GFX1200-NEXT: v_cndmask_b16 v0.l, 0x7f, v0.l, s0
+; GFX1200-NEXT: v_or_b16 v6.l, v1.h, v3.l
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_or_b16 v6.l, v1.l, v0.h
-; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_or_b16 v0.l, v1.h, v2.l
-; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v2
+; GFX1200-NEXT: v_and_b16 v0.h, v1.l, 0xff
+; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v0.l
+; GFX1200-NEXT: v_mov_b16_e32 v1.h, v2.l
; GFX1200-NEXT: v_lshrrev_b64 v[3:4], 24, v[5:6]
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_or_b16 v0.l, v0.h, v1.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX1200-NEXT: v_lshrrev_b32_e32 v5, 8, v6
-; GFX1200-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX1200-NEXT: v_mov_b16_e32 v4.l, v6.l
; GFX1200-NEXT: v_mov_b16_e32 v6.l, v6.h
; GFX1200-NEXT: s_setpc_b64 s[30:31]
@@ -2782,21 +2763,20 @@ define <7 x i8> @to_fp8_v7f16(<7 x half> %x) {
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v4.h, v1
+; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v4.l, v1
; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v7.h, v3
; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v7.l, v2
-; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v4.l, v0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.h
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX1250-TRUE16-NEXT: v_dual_lshrrev_b32 v5, 8, v7 :: v_dual_lshlrev_b32 v6, 16, v1
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v4
+; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v0.l, v0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
+; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v7
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.l
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.l, v7.l
; GFX1250-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[6:7]
+; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v6.l, v7.h
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
@@ -3239,437 +3219,445 @@ define <5 x i8> @to_bf8_v5f16(<5 x half> %x) {
; GFX1200-NEXT: s_wait_kmcnt 0x0
; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v3.l, v2.l
; GFX1200-NEXT: v_frexp_mant_f16_e32 v3.h, v2.l
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v4.h, v2.h
; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v5.h, v2.h
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.l, v2.h
; GFX1200-NEXT: v_lshrrev_b16 v4.l, 8, v2.l
-; GFX1200-NEXT: v_sub_nc_u16 v5.l, -5, v3.l
+; GFX1200-NEXT: v_sub_nc_u16 v4.h, -5, v3.l
; GFX1200-NEXT: v_and_b16 v6.h, v3.h, 0x3ff
; GFX1200-NEXT: v_and_b16 v7.l, v3.h, 0x7f
-; GFX1200-NEXT: v_sub_nc_u16 v8.l, -5, v5.h
-; GFX1200-NEXT: v_and_b16 v9.l, v4.h, 0x7f
-; GFX1200-NEXT: v_min_u16 v5.l, v5.l, 15
-; GFX1200-NEXT: v_or_b16 v9.h, v6.h, 0x400
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, 8, v6.h
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.l
; GFX1200-NEXT: v_lshrrev_b16 v3.h, 7, v3.h
-; GFX1200-NEXT: v_sub_nc_u16 v8.h, v5.l, 1 clamp
-; GFX1200-NEXT: v_min_u16 v8.l, v8.l, 15
-; GFX1200-NEXT: v_and_b16 v11.l, v6.h, 1
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v9.l
-; GFX1200-NEXT: v_lshlrev_b16 v10.l, v8.h, 1
-; GFX1200-NEXT: v_lshrrev_b16 v9.l, v5.l, v9.h
-; GFX1200-NEXT: v_and_b16 v7.h, v4.h, 0x3ff
-; GFX1200-NEXT: v_or_b16 v11.l, v12.l, v11.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc_lo
-; GFX1200-NEXT: v_add_nc_u16 v10.l, v10.l, -1
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.l
-; GFX1200-NEXT: v_sub_nc_u16 v5.l, v8.l, 1 clamp
-; GFX1200-NEXT: v_and_b16 v3.h, v3.h, v11.l
-; GFX1200-NEXT: v_and_b16 v11.l, v9.l, 1
-; GFX1200-NEXT: v_and_b16 v10.l, v9.h, v10.l
-; GFX1200-NEXT: v_or_b16 v10.h, v7.h, 0x400
-; GFX1200-NEXT: v_lshlrev_b16 v11.h, v5.l, 1
-; GFX1200-NEXT: v_add_nc_u16 v3.h, v6.h, v3.h
-; GFX1200-NEXT: v_lshrrev_b16 v8.h, v8.h, v9.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v10.l
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, 8, v7.h
-; GFX1200-NEXT: v_lshrrev_b16 v10.l, v8.l, v10.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v8.l
-; GFX1200-NEXT: v_lshrrev_b16 v4.h, 7, v4.h
+; GFX1200-NEXT: v_sub_nc_u16 v8.h, -5, v5.h
+; GFX1200-NEXT: v_min_u16 v4.h, v4.h, 15
+; GFX1200-NEXT: v_or_b16 v10.h, v6.h, 0x400
+; GFX1200-NEXT: v_lshrrev_b16 v6.h, 8, v6.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.l
+; GFX1200-NEXT: v_and_b16 v9.l, v5.l, 0x7f
+; GFX1200-NEXT: v_sub_nc_u16 v8.l, v4.h, 1 clamp
+; GFX1200-NEXT: v_min_u16 v8.h, v8.h, 15
+; GFX1200-NEXT: v_and_b16 v11.h, v6.h, 1
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v3.h
-; GFX1200-NEXT: v_and_b16 v6.h, v7.h, 1
+; GFX1200-NEXT: v_lshrrev_b16 v12.h, v4.h, v10.h
+; GFX1200-NEXT: v_lshlrev_b16 v10.l, v8.l, 1
+; GFX1200-NEXT: v_and_b16 v7.h, v5.l, 0x3ff
+; GFX1200-NEXT: v_lshrrev_b16 v5.l, 7, v5.l
+; GFX1200-NEXT: v_or_b16 v11.h, v12.l, v11.h
; GFX1200-NEXT: v_and_b16 v4.l, v4.l, 0x80
-; GFX1200-NEXT: v_lshrrev_b16 v5.l, v5.l, v10.h
-; GFX1200-NEXT: v_or_b16 v9.h, v12.l, v11.l
-; GFX1200-NEXT: v_add_nc_u16 v11.l, v11.h, -1
+; GFX1200-NEXT: v_add_nc_u16 v7.l, v10.l, -1
+; GFX1200-NEXT: v_or_b16 v11.l, v7.h, 0x400
+; GFX1200-NEXT: v_lshrrev_b16 v7.h, 8, v7.h
+; GFX1200-NEXT: v_and_b16 v3.h, v3.h, v11.h
+; GFX1200-NEXT: v_sub_nc_u16 v11.h, v8.h, 1 clamp
+; GFX1200-NEXT: v_and_b16 v7.l, v10.h, v7.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v8.h
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v6.l, v1.l
+; GFX1200-NEXT: v_add_nc_u16 v3.h, v6.h, v3.h
+; GFX1200-NEXT: v_or_b16 v10.l, v4.l, 0x7c
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.l
+; GFX1200-NEXT: v_and_b16 v7.l, v12.h, 1
+; GFX1200-NEXT: v_sub_nc_u16 v9.h, -5, v6.l
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 3, v3.h
+; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v2.l
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v9.l
+; GFX1200-NEXT: v_cmp_class_f16_e64 s5, v2.l, 0x204
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s1
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1200-NEXT: v_mov_b16_e32 v9.l, v13.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, s0
-; GFX1200-NEXT: v_mov_b16_e32 v12.l, v13.l
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s0
-; GFX1200-NEXT: v_and_b16 v8.h, v8.h, v9.h
-; GFX1200-NEXT: v_and_b16 v9.h, v10.h, v11.l
-; GFX1200-NEXT: v_mov_b16_e32 v8.l, v14.l
-; GFX1200-NEXT: v_or_b16 v6.h, v12.l, v6.h
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v6.l, v1.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v4.h
+; GFX1200-NEXT: v_lshrrev_b16 v4.h, v8.l, v10.h
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s1
+; GFX1200-NEXT: v_or_b16 v6.h, v9.l, v7.l
+; GFX1200-NEXT: v_lshlrev_b16 v7.l, v11.h, 1
+; GFX1200-NEXT: v_mov_b16_e32 v12.l, v14.l
+; GFX1200-NEXT: v_lshrrev_b16 v8.l, v8.h, v11.l
+; GFX1200-NEXT: v_mov_b16_e32 v9.l, v13.l
+; GFX1200-NEXT: v_and_b16 v4.h, v4.h, v6.h
+; GFX1200-NEXT: v_add_nc_u16 v6.h, v7.l, -1
+; GFX1200-NEXT: v_and_b16 v7.l, v7.h, 1
+; GFX1200-NEXT: v_and_b16 v8.h, v8.l, 1
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, v9.l
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v4.h, s0
+; GFX1200-NEXT: v_and_b16 v6.h, v11.l, v6.h
+; GFX1200-NEXT: v_or_b16 v7.l, v12.l, v7.l
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 3, v3.h
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, 14
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v12.h, v4.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.h
+; GFX1200-NEXT: v_and_b16 v5.l, v5.l, v7.l
+; GFX1200-NEXT: v_lshrrev_b16 v6.h, v11.h, v11.l
+; GFX1200-NEXT: v_lshlrev_b16 v10.h, 2, v3.l
+; GFX1200-NEXT: v_cmp_eq_u16_e64 s3, 30, v3.l
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v4.h
+; GFX1200-NEXT: v_add_nc_u16 v5.l, v7.h, v5.l
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s4, 30, v3.l
+; GFX1200-NEXT: s_and_b32 s1, s3, s1
+; GFX1200-NEXT: v_mov_b16_e32 v9.l, v13.l
+; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, 4, s0
+; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s0
+; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v3.l
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: s_or_b32 s1, s4, s1
+; GFX1200-NEXT: v_or_b16 v8.h, v9.l, v8.h
+; GFX1200-NEXT: v_or_b16 v9.l, v4.l, v10.h
+; GFX1200-NEXT: v_or_b16 v7.l, v4.l, v7.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_and_b16 v6.h, v6.h, v8.h
+; GFX1200-NEXT: v_or_b16 v3.h, v9.l, v3.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_or_b16 v4.h, v7.l, v4.h
+; GFX1200-NEXT: v_cndmask_b16 v3.l, 0, v6.h, s2
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.h, v1.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v4.h, s0
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v5.l
+; GFX1200-NEXT: v_min_u16 v4.h, v9.h, 15
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v8.l, v3.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v2
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v10.l, s1
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0
+; GFX1200-NEXT: v_sub_nc_u16 v7.h, v4.h, 1 clamp
+; GFX1200-NEXT: v_and_b16 v9.h, v6.h, 0x7f
+; GFX1200-NEXT: v_lshlrev_b16 v8.h, 7, v9.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v4.l, vcc_lo
+; GFX1200-NEXT: v_add_nc_u16 v4.l, v5.h, v7.l
+; GFX1200-NEXT: v_and_b16 v5.h, v6.h, 0x3ff
+; GFX1200-NEXT: v_lshlrev_b16 v7.l, v7.h, 1
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v3.l
+; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, s0
+; GFX1200-NEXT: v_add_nc_u16 v4.l, v4.l, 14
+; GFX1200-NEXT: v_or_b16 v9.l, v5.h, 0x400
+; GFX1200-NEXT: v_add_nc_u16 v7.l, v7.l, -1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v8.h, 0, v8.h, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 4, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, vcc_lo
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, 8, v5.h
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v9.h
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, v8.l
-; GFX1200-NEXT: v_and_b16 v4.h, v4.h, v6.h
-; GFX1200-NEXT: v_and_b16 v6.h, v10.l, 1
-; GFX1200-NEXT: v_add_nc_u16 v8.l, v9.l, v8.h
+; GFX1200-NEXT: v_and_b16 v7.l, v9.l, v7.l
+; GFX1200-NEXT: v_lshlrev_b16 v10.h, 2, v4.l
+; GFX1200-NEXT: v_or_b16 v8.l, v8.h, v8.l
+; GFX1200-NEXT: v_lshrrev_b16 v6.h, 7, v6.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, 14
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v7.h, v4.h
-; GFX1200-NEXT: v_or_b16 v7.l, v4.l, 0x7c
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v8.l
-; GFX1200-NEXT: v_mov_b16_e32 v9.l, v11.l
-; GFX1200-NEXT: v_lshlrev_b16 v7.h, 2, v3.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 3, v4.h
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v3.l
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.l
+; GFX1200-NEXT: v_and_b16 v7.l, v5.h, 1
+; GFX1200-NEXT: v_or_b16 v10.h, v8.h, v10.h
+; GFX1200-NEXT: v_lshrrev_b16 v9.h, v4.h, v9.l
+; GFX1200-NEXT: v_or_b16 v3.l, v8.l, v3.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v8.h, 0, 4, vcc_lo
-; GFX1200-NEXT: v_or_b16 v6.h, v9.l, v6.h
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 30, v3.l
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, s2
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s2
-; GFX1200-NEXT: v_and_b16 v5.l, v5.l, v6.h
-; GFX1200-NEXT: v_or_b16 v6.h, v4.l, v7.h
-; GFX1200-NEXT: v_cndmask_b16 v7.h, v8.l, 0, vcc_lo
-; GFX1200-NEXT: v_or_b16 v8.l, v4.l, v8.h
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v3.h
-; GFX1200-NEXT: v_cndmask_b16 v5.l, 0, v5.l, s1
-; GFX1200-NEXT: v_or_b16 v3.h, v6.h, v3.h
-; GFX1200-NEXT: v_cmp_eq_u16_e64 s1, 30, v3.l
-; GFX1200-NEXT: v_or_b16 v6.h, v8.l, v7.h
-; GFX1200-NEXT: v_mov_b16_e32 v8.l, v9.l
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v10.l, v5.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v6.h, s0
-; GFX1200-NEXT: s_and_b32 s0, s1, vcc_lo
-; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.h, v8.l
+; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo
+; GFX1200-NEXT: v_or_b16 v7.l, v11.l, v7.l
+; GFX1200-NEXT: v_or_b16 v8.l, v10.h, v5.l
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.l
+; GFX1200-NEXT: v_and_b16 v11.h, v9.h, 1
+; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v2.l, v2.l
+; GFX1200-NEXT: v_and_b16 v6.h, v6.h, v7.l
+; GFX1200-NEXT: v_lshrrev_b16 v7.l, v7.h, v9.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v2.l, v8.l, v3.l, vcc_lo
+; GFX1200-NEXT: v_or_b16 v10.h, v12.l, v11.h
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v4.h
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v5.h, v6.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 3, v5.l
+; GFX1200-NEXT: v_cmp_eq_u16_e64 s2, 30, v4.l
+; GFX1200-NEXT: v_and_b16 v5.h, v7.l, v10.h
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v10.l, s5
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 3, v3.l
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v8.l, v1.h
+; GFX1200-NEXT: s_and_b32 s1, s2, s1
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v5.h, vcc_lo
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 30, v4.l
+; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, s3
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v5.h, v1.h
+; GFX1200-NEXT: v_or_b16 v4.l, v8.h, 0x7c
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v9.h, v4.h
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_or_b32 s0, s3, s0
-; GFX1200-NEXT: v_sub_nc_u16 v5.h, -5, v6.l
+; GFX1200-NEXT: s_or_b32 s1, vcc_lo, s1
+; GFX1200-NEXT: v_mov_b16_e32 v5.l, v7.l
+; GFX1200-NEXT: v_sub_nc_u16 v7.l, -5, v5.h
+; GFX1200-NEXT: v_cndmask_b16 v3.h, 0x7e, v3.h, s0
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v4.h
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v7.l, s0
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v2.l
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v3.l
-; GFX1200-NEXT: v_lshrrev_b32_e32 v8, 31, v2
-; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, 14
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v4.l, s0
-; GFX1200-NEXT: v_min_u16 v4.l, v5.h, 15
+; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v4.l, s1
+; GFX1200-NEXT: v_add_nc_u16 v5.l, v6.l, v5.l
+; GFX1200-NEXT: v_lshrrev_b16 v6.l, 8, v1.l
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v2.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 4, vcc_lo
-; GFX1200-NEXT: v_lshlrev_b16 v7.h, 7, v8.l
-; GFX1200-NEXT: v_lshlrev_b16 v5.h, 2, v5.l
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v8.l, v1.l
-; GFX1200-NEXT: v_sub_nc_u16 v8.h, v4.l, 1 clamp
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, vcc_lo
-; GFX1200-NEXT: v_or_b16 v6.h, v7.h, v6.h
-; GFX1200-NEXT: v_or_b16 v5.h, v7.h, v5.h
-; GFX1200-NEXT: v_and_b16 v9.l, v8.l, 0x3ff
-; GFX1200-NEXT: v_lshlrev_b16 v9.h, v8.h, 1
-; GFX1200-NEXT: v_cmp_class_f16_e64 s0, v2.l, 0x204
-; GFX1200-NEXT: v_or_b16 v3.l, v6.h, v3.l
-; GFX1200-NEXT: v_or_b16 v5.h, v5.h, v4.h
+; GFX1200-NEXT: v_min_u16 v7.l, v7.l, 15
+; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, 14
+; GFX1200-NEXT: v_and_b16 v6.l, v6.l, 0x80
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v8.h, s0
+; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s3
+; GFX1200-NEXT: v_lshlrev_b16 v7.h, 2, v5.l
+; GFX1200-NEXT: v_or_b16 v6.h, v6.l, v6.h
+; GFX1200-NEXT: v_sub_nc_u16 v8.h, v7.l, 1 clamp
+; GFX1200-NEXT: v_cmp_class_f16_e64 s2, v2.h, 0x204
; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v5.l
-; GFX1200-NEXT: v_or_b16 v6.h, v9.l, 0x400
-; GFX1200-NEXT: v_add_nc_u16 v9.h, v9.h, -1
-; GFX1200-NEXT: v_and_b16 v10.l, v8.l, 0x7f
+; GFX1200-NEXT: v_or_b16 v7.h, v6.l, v7.h
+; GFX1200-NEXT: v_or_b16 v4.h, v6.h, v4.h
+; GFX1200-NEXT: v_lshlrev_b16 v9.l, v8.h, 1
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v3.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v7.l, s0
+; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v4.l, s2
+; GFX1200-NEXT: v_or_b16 v6.h, v7.h, v3.l
+; GFX1200-NEXT: v_and_b16 v7.h, v8.l, 0x3ff
+; GFX1200-NEXT: v_add_nc_u16 v9.l, v9.l, -1
+; GFX1200-NEXT: v_cmp_eq_u16_e64 s1, 30, v5.l
+; GFX1200-NEXT: v_or_b16 v9.h, v6.l, 0x7c
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v5.h, v3.l, vcc_lo
-; GFX1200-NEXT: v_lshrrev_b16 v7.l, 8, v9.l
-; GFX1200-NEXT: v_and_b16 v5.h, v6.h, v9.h
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v10.l
-; GFX1200-NEXT: v_lshrrev_b16 v9.l, v4.l, v6.h
-; GFX1200-NEXT: v_lshrrev_b16 v8.l, 7, v8.l
-; GFX1200-NEXT: v_cmp_eq_u16_e64 s0, 30, v5.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 30, v5.l
+; GFX1200-NEXT: v_cndmask_b16 v4.l, v6.h, v4.h, vcc_lo
+; GFX1200-NEXT: v_or_b16 v3.l, v7.h, 0x400
+; GFX1200-NEXT: v_and_b16 v4.h, v8.l, 0x7f
+; GFX1200-NEXT: v_lshrrev_b16 v7.h, 8, v7.h
+; GFX1200-NEXT: s_and_b32 s0, s1, s0
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v7.l
+; GFX1200-NEXT: v_and_b16 v6.h, v3.l, v9.l
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v4.h
+; GFX1200-NEXT: v_and_b16 v4.h, v7.h, 1
+; GFX1200-NEXT: v_lshrrev_b16 v9.l, v7.l, v3.l
+; GFX1200-NEXT: v_lshrrev_b16 v3.l, v8.h, v3.l
+; GFX1200-NEXT: v_and_b16 v3.h, v3.h, 0xff
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.h
-; GFX1200-NEXT: v_and_b16 v5.h, v7.l, 1
-; GFX1200-NEXT: v_lshrrev_b16 v5.l, v8.h, v6.h
-; GFX1200-NEXT: v_or_b16 v6.h, v7.h, 0x7c
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v6.h
+; GFX1200-NEXT: v_and_b16 v6.h, v9.l, 1
+; GFX1200-NEXT: v_cmp_class_f16_e64 s5, v1.h, 0x204
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v4.h
-; GFX1200-NEXT: v_and_b16 v4.h, v9.l, 1
-; GFX1200-NEXT: v_or_b16 v5.h, v10.l, v5.h
-; GFX1200-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_or_b16 v4.h, v11.l, v4.h
-; GFX1200-NEXT: v_and_b16 v5.h, v8.l, v5.h
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v4.l
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 30, v5.l
+; GFX1200-NEXT: v_mov_b16_e32 v5.l, v10.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1200-NEXT: v_mov_b16_e32 v10.l, v11.l
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_or_b32 s1, s1, s0
-; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v2.l, v2.l
-; GFX1200-NEXT: v_and_b16 v4.h, v5.l, v4.h
-; GFX1200-NEXT: v_add_nc_u16 v4.l, v7.l, v5.h
+; GFX1200-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX1200-NEXT: v_or_b16 v4.h, v5.l, v4.h
+; GFX1200-NEXT: v_lshrrev_b16 v5.l, 7, v8.l
+; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v2.h, v2.h
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.l, v3.l, v6.h, s1
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v3.l, v1.h
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v2.h
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v4.l, v9.h, s0
+; GFX1200-NEXT: v_or_b16 v6.h, v10.l, v6.h
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v1.l
+; GFX1200-NEXT: v_and_b16 v4.l, v5.l, v4.h
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v4.h, v0.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v4.h, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7e, v2.l, vcc_lo
+; GFX1200-NEXT: v_and_b16 v3.l, v3.l, v6.h
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v6.l, s0
+; GFX1200-NEXT: v_add_nc_u16 v4.l, v7.h, v4.l
+; GFX1200-NEXT: v_sub_nc_u16 v5.l, -5, v4.h
+; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 31, v1
+; GFX1200-NEXT: v_cndmask_b16 v3.l, 0, v3.l, s1
+; GFX1200-NEXT: v_cmp_class_f16_e64 s0, v1.l, 0x204
; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v4.l
+; GFX1200-NEXT: v_min_u16 v5.l, v5.l, 15
+; GFX1200-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v9.l, v3.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.h, 0x7e, v3.h, s0
-; GFX1200-NEXT: v_sub_nc_u16 v5.l, -5, v3.l
-; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v7.h, s1
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v9.l, v4.h
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v9.h, s0
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, 8, v1.l
-; GFX1200-NEXT: v_min_u16 v5.l, v5.l, 15
-; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, 0, vcc_lo
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v4.h
-; GFX1200-NEXT: v_mov_b16_e32 v7.l, v8.l
-; GFX1200-NEXT: v_and_b16 v7.h, v7.h, 0x80
-; GFX1200-NEXT: v_cmp_class_f16_e64 s1, v2.h, 0x204
-; GFX1200-NEXT: v_and_b16 v3.h, v3.h, 0xff
+; GFX1200-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
+; GFX1200-NEXT: v_sub_nc_u16 v6.h, v5.l, 1 clamp
+; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v1.l, v1.l
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v3.l
+; GFX1200-NEXT: v_lshlrev_b16 v7.l, 7, v7.l
+; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, v6.l
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v0.l
+; GFX1200-NEXT: v_lshlrev_b16 v8.l, v6.h, 1
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 4, s0
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v6.l, v7.l
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v1.h
-; GFX1200-NEXT: v_sub_nc_u16 v7.l, v5.l, 1 clamp
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s0
-; GFX1200-NEXT: v_or_b16 v8.l, v7.h, v8.l
+; GFX1200-NEXT: v_cndmask_b16 v1.l, 0, 4, s0
+; GFX1200-NEXT: v_cndmask_b16 v9.l, v4.l, 0, vcc_lo
; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, 14
-; GFX1200-NEXT: v_and_b16 v8.h, v6.l, 0x3ff
-; GFX1200-NEXT: v_lshlrev_b16 v9.l, v7.l, 1
-; GFX1200-NEXT: v_and_b16 v10.h, v6.l, 0x7f
-; GFX1200-NEXT: v_or_b16 v4.h, v8.l, v4.h
-; GFX1200-NEXT: v_lshlrev_b16 v9.h, 2, v5.h
-; GFX1200-NEXT: v_or_b16 v10.l, v8.h, 0x400
-; GFX1200-NEXT: v_add_nc_u16 v9.l, v9.l, -1
-; GFX1200-NEXT: v_lshrrev_b16 v8.h, 8, v8.h
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v10.h
-; GFX1200-NEXT: v_or_b16 v9.h, v7.h, v9.h
-; GFX1200-NEXT: v_lshrrev_b16 v10.h, v5.l, v10.l
-; GFX1200-NEXT: v_and_b16 v9.l, v10.l, v9.l
+; GFX1200-NEXT: v_and_b16 v7.h, v6.l, 0x3ff
+; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, -1
+; GFX1200-NEXT: v_or_b16 v4.l, v3.h, v2.l
+; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7e, v2.h, s1
+; GFX1200-NEXT: v_lshlrev_b16 v8.h, 2, v5.h
+; GFX1200-NEXT: v_or_b16 v9.h, v7.h, 0x400
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s0
+; GFX1200-NEXT: v_or_b16 v1.l, v7.l, v1.l
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v9.l
+; GFX1200-NEXT: v_or_b16 v8.h, v7.l, v8.h
+; GFX1200-NEXT: v_and_b16 v2.h, v9.h, v8.l
+; GFX1200-NEXT: v_lshrrev_b16 v3.h, v5.l, v9.h
+; GFX1200-NEXT: v_or_b16 v1.l, v1.l, v3.l
+; GFX1200-NEXT: v_cmp_eq_u16_e64 s0, 30, v5.h
+; GFX1200-NEXT: v_or_b16 v3.l, v8.h, v9.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v2.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 30, v5.h
+; GFX1200-NEXT: v_and_b16 v2.h, v3.h, 1
+; GFX1200-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX1200-NEXT: v_lshrrev_b16 v6.h, v6.h, v9.h
+; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, s1
+; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v5.h
+; GFX1200-NEXT: v_and_b16 v5.h, v6.l, 0x7f
+; GFX1200-NEXT: v_lshrrev_b16 v7.h, 8, v7.h
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v8.h, v0.h
+; GFX1200-NEXT: v_or_b16 v2.h, v8.l, v2.h
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s1
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v3.l, v0.h
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.h
+; GFX1200-NEXT: v_and_b16 v5.h, v7.h, 1
+; GFX1200-NEXT: v_and_b16 v2.h, v6.h, v2.h
; GFX1200-NEXT: v_lshrrev_b16 v6.l, 7, v6.l
+; GFX1200-NEXT: v_sub_nc_u16 v8.l, -5, v3.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
-; GFX1200-NEXT: v_or_b16 v9.h, v9.h, v4.l
-; GFX1200-NEXT: v_and_b16 v8.l, v10.h, 1
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v9.l
-; GFX1200-NEXT: v_and_b16 v9.l, v8.h, 1
-; GFX1200-NEXT: v_lshrrev_b16 v7.l, v7.l, v10.l
-; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v2.h, v2.h
-; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v6.h, s1
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo
-; GFX1200-NEXT: v_or_b16 v9.l, v11.l, v9.l
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v5.h
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 3, v4.l
-; GFX1200-NEXT: v_cmp_eq_u16_e64 s2, 30, v5.h
-; GFX1200-NEXT: v_or_b16 v8.l, v12.l, v8.l
-; GFX1200-NEXT: v_and_b16 v6.l, v6.l, v9.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v9.h, v4.h, vcc_lo
+; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc_lo
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 30, v5.h
-; GFX1200-NEXT: v_and_b16 v4.h, v7.l, v8.l
-; GFX1200-NEXT: v_add_nc_u16 v5.l, v8.h, v6.l
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v5.h, v0.l
-; GFX1200-NEXT: s_and_b32 s1, s2, s1
-; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7e, v2.l, s0
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v4.l, 0, v4.h, vcc_lo
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v5.l
-; GFX1200-NEXT: v_or_b16 v4.h, v7.h, 0x7c
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_or_b32 s1, s3, s1
-; GFX1200-NEXT: v_lshrrev_b32_e32 v8, 31, v1
-; GFX1200-NEXT: v_add_nc_u16 v4.l, v10.h, v4.l
+; GFX1200-NEXT: s_or_b32 s0, s2, s0
+; GFX1200-NEXT: v_or_b16 v6.h, v7.l, 0x7c
+; GFX1200-NEXT: v_min_u16 v5.l, v8.l, 15
+; GFX1200-NEXT: v_mov_b16_e32 v8.l, v9.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v4.h, s1
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v1.l
-; GFX1200-NEXT: v_lshlrev_b16 v7.l, 7, v8.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v4.l
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, v6.l
-; GFX1200-NEXT: v_sub_nc_u16 v6.l, -5, v5.h
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v7.h, s1
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v8.l, v0.l
-; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 4, s0
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, 14
-; GFX1200-NEXT: v_min_u16 v6.l, v6.l, 15
-; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, 0, s0
-; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, vcc_lo
-; GFX1200-NEXT: v_or_b16 v6.h, v7.l, v6.h
-; GFX1200-NEXT: v_lshlrev_b16 v7.h, 2, v3.l
-; GFX1200-NEXT: v_sub_nc_u16 v8.h, v6.l, 1 clamp
-; GFX1200-NEXT: v_and_b16 v9.l, v8.l, 0x3ff
-; GFX1200-NEXT: v_cmp_class_f16_e64 s0, v1.l, 0x204
-; GFX1200-NEXT: v_or_b16 v4.l, v6.h, v4.l
-; GFX1200-NEXT: v_or_b16 v7.h, v7.l, v7.h
-; GFX1200-NEXT: v_lshlrev_b16 v9.h, v8.h, 1
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v3.l
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v4.h, s0
-; GFX1200-NEXT: v_cmp_eq_u16_e64 s0, 30, v3.l
-; GFX1200-NEXT: v_or_b16 v6.h, v7.h, v5.l
-; GFX1200-NEXT: v_or_b16 v7.h, v9.l, 0x400
-; GFX1200-NEXT: v_add_nc_u16 v9.h, v9.h, -1
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 30, v3.l
-; GFX1200-NEXT: v_cmp_class_f16_e64 s3, v1.h, 0x204
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v4.l, v6.h, v4.l, vcc_lo
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v5.l
-; GFX1200-NEXT: v_and_b16 v4.h, v7.h, v9.h
-; GFX1200-NEXT: v_lshrrev_b16 v3.l, v6.l, v7.h
-; GFX1200-NEXT: v_and_b16 v6.h, v8.l, 0x7f
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, v8.h, v7.h
-; GFX1200-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v4.h
-; GFX1200-NEXT: v_or_b16 v4.h, v7.l, 0x7c
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_or_b32 s0, s1, s0
-; GFX1200-NEXT: v_and_b16 v5.l, v3.l, 1
-; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v1.l, v1.l
-; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s2
+; GFX1200-NEXT: v_cndmask_b16 v2.h, 0, v2.h, vcc_lo
+; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v1.h
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, v4.l, v4.h, s0
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v4.l, v0.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.h
-; GFX1200-NEXT: v_lshrrev_b16 v8.h, 8, v9.l
-; GFX1200-NEXT: v_or_b16 v5.l, v10.l, v5.l
-; GFX1200-NEXT: v_lshrrev_b16 v8.l, 7, v8.l
-; GFX1200-NEXT: v_sub_nc_u16 v9.l, -5, v4.l
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.l
-; GFX1200-NEXT: v_and_b16 v5.l, v7.h, v5.l
-; GFX1200-NEXT: v_and_b16 v6.h, v8.h, 1
-; GFX1200-NEXT: v_min_u16 v6.l, v9.l, 15
-; GFX1200-NEXT: v_mov_b16_e32 v9.l, v10.l
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v7.h, v0.h
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v5.l, 0, v5.l, s0
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v1.h
-; GFX1200-NEXT: v_sub_nc_u16 v9.h, v6.l, 1 clamp
-; GFX1200-NEXT: v_or_b16 v6.h, v9.l, v6.h
-; GFX1200-NEXT: v_and_b16 v9.l, v7.h, 0x3ff
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, v5.l
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v7.l, s1
-; GFX1200-NEXT: v_lshlrev_b16 v10.l, v9.h, 1
-; GFX1200-NEXT: v_and_b16 v5.l, v8.l, v6.h
-; GFX1200-NEXT: v_and_b16 v8.l, v7.h, 0x7f
-; GFX1200-NEXT: v_or_b16 v6.h, v9.l, 0x400
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v7.h
-; GFX1200-NEXT: v_add_nc_u16 v7.l, v10.l, -1
-; GFX1200-NEXT: v_add_nc_u16 v5.l, v8.h, v5.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v8.l
-; GFX1200-NEXT: v_lshrrev_b16 v8.h, 8, v9.l
-; GFX1200-NEXT: v_lshrrev_b16 v8.l, v6.l, v6.h
-; GFX1200-NEXT: v_and_b16 v7.l, v6.h, v7.l
-; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v4.h, s3
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v6.h, s0
+; GFX1200-NEXT: v_sub_nc_u16 v9.l, v5.l, 1 clamp
+; GFX1200-NEXT: v_or_b16 v5.h, v8.l, v5.h
+; GFX1200-NEXT: v_and_b16 v8.l, v8.h, 0x3ff
+; GFX1200-NEXT: v_add_nc_u16 v2.h, v3.h, v2.h
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v7.l, vcc_lo
+; GFX1200-NEXT: v_lshlrev_b16 v9.h, v9.l, 1
+; GFX1200-NEXT: v_and_b16 v3.h, v6.l, v5.h
+; GFX1200-NEXT: v_or_b16 v5.h, v8.l, 0x400
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v2.h
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.l
+; GFX1200-NEXT: v_add_nc_u16 v6.l, v9.h, -1
+; GFX1200-NEXT: v_and_b16 v9.h, v8.h, 0x7f
+; GFX1200-NEXT: v_add_nc_u16 v3.h, v7.h, v3.h
+; GFX1200-NEXT: v_lshrrev_b16 v7.h, 8, v8.l
+; GFX1200-NEXT: v_lshrrev_b16 v8.l, v5.l, v5.h
+; GFX1200-NEXT: v_and_b16 v6.l, v5.h, v6.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v9.h
+; GFX1200-NEXT: v_lshrrev_b16 v8.h, 7, v8.h
+; GFX1200-NEXT: v_and_b16 v9.h, v7.h, 1
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, v9.l, v5.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v6.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s1
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 3, v5.l
-; GFX1200-NEXT: v_and_b16 v9.l, v8.h, 1
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v7.l
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 3, v3.h
; GFX1200-NEXT: v_and_b16 v10.h, v8.l, 1
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v3.l
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, s0
+; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s2
+; GFX1200-NEXT: v_or_b16 v9.h, v10.l, v9.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s1
-; GFX1200-NEXT: v_or_b16 v9.l, v10.l, v9.l
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s2
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, v9.h, v6.h
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s0
-; GFX1200-NEXT: v_mov_b16_e32 v7.l, v11.l
-; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, s1
+; GFX1200-NEXT: v_lshrrev_b16 v9.l, 8, v0.l
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s1
; GFX1200-NEXT: v_mov_b16_e32 v10.l, v12.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.h, 0x7e, v2.h, vcc_lo
-; GFX1200-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v5.h, v7.l
-; GFX1200-NEXT: v_and_b16 v5.h, v7.h, v9.l
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v6.h, s5
+; GFX1200-NEXT: v_mov_b16_e32 v6.l, v11.l
+; GFX1200-NEXT: v_and_b16 v5.l, v9.l, 0x80
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 3, v3.h
; GFX1200-NEXT: v_or_b16 v7.l, v10.l, v10.h
-; GFX1200-NEXT: v_lshrrev_b16 v9.l, 8, v0.l
-; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, 4, s0
+; GFX1200-NEXT: v_and_b16 v2.l, v2.l, 0xff
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, v6.l
+; GFX1200-NEXT: v_and_b16 v6.l, v8.h, v9.h
+; GFX1200-NEXT: v_cndmask_b16 v8.h, 0, 4, s0
+; GFX1200-NEXT: v_and_b16 v5.h, v5.h, v7.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, 14
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v8.h, v5.h
-; GFX1200-NEXT: v_and_b16 v6.h, v6.h, v7.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.l
-; GFX1200-NEXT: v_and_b16 v6.l, v9.l, 0x80
+; GFX1200-NEXT: v_add_nc_u16 v6.l, v7.h, v6.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1200-NEXT: v_or_b16 v7.h, v5.l, v8.h
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, v5.h, vcc_lo
; GFX1200-NEXT: v_lshlrev_b16 v7.l, 2, v4.h
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 3, v5.h
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s4, 3, v5.l
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v6.l
+; GFX1200-NEXT: v_or_b16 v2.h, v7.h, v2.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1200-NEXT: v_add_nc_u16 v5.h, v8.l, v5.h
+; GFX1200-NEXT: v_lshrrev_b32_e32 v8, 31, v0
+; GFX1200-NEXT: v_or_b16 v7.l, v5.l, v7.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, v6.h, s0
-; GFX1200-NEXT: v_or_b16 v7.h, v6.l, v7.h
-; GFX1200-NEXT: v_or_b16 v7.l, v6.l, v7.l
-; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, s2
-; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v1.h, v1.h
-; GFX1200-NEXT: v_add_nc_u16 v6.h, v8.l, v6.h
-; GFX1200-NEXT: v_or_b16 v1.h, v7.h, v3.l
-; GFX1200-NEXT: v_cmp_eq_u16_e64 s5, 30, v4.h
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v4.l, v9.l
-; GFX1200-NEXT: v_or_b16 v4.l, v7.l, v5.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 3, v6.h
-; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 31, v0
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s3, 1, v4.h
+; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, s0
+; GFX1200-NEXT: v_cmp_eq_u16_e64 s3, 30, v4.h
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v5.h
+; GFX1200-NEXT: v_lshlrev_b16 v7.h, 7, v8.l
+; GFX1200-NEXT: v_or_b16 v7.l, v7.l, v3.h
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, v9.l
+; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v4.h
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v3.h, 0, 4, vcc_lo
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s4, 30, v4.h
+; GFX1200-NEXT: v_cndmask_b16 v4.h, v5.h, 0, vcc_lo
; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, 14
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s6, 30, v4.h
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v5.l, 0, 4, s1
-; GFX1200-NEXT: v_lshlrev_b16 v7.l, 7, v7.l
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v6.h, 0, s1
-; GFX1200-NEXT: v_lshlrev_b16 v7.h, 2, v3.l
-; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, s2
-; GFX1200-NEXT: s_and_b32 s1, s5, s4
-; GFX1200-NEXT: v_or_b16 v5.l, v7.l, v5.l
-; GFX1200-NEXT: v_cndmask_b16 v1.h, v4.l, v1.h, s3
-; GFX1200-NEXT: v_or_b16 v6.h, v7.l, v7.h
+; GFX1200-NEXT: v_cndmask_b16 v5.h, v6.l, 0, s0
+; GFX1200-NEXT: v_or_b16 v3.h, v7.h, v3.h
+; GFX1200-NEXT: s_and_b32 s0, s3, s2
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v7.l, v2.h, s1
+; GFX1200-NEXT: v_lshlrev_b16 v8.l, 2, v3.l
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v3.l
+; GFX1200-NEXT: v_or_b16 v3.h, v3.h, v4.h
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_or_b32 s5, s6, s1
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v3.l
-; GFX1200-NEXT: v_or_b16 v4.l, v5.l, v4.h
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 3, v5.h
-; GFX1200-NEXT: v_or_b16 v4.h, v6.h, v5.h
-; GFX1200-NEXT: v_cmp_eq_u16_e64 s3, 30, v3.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s4, 30, v3.l
-; GFX1200-NEXT: v_or_b16 v5.l, v6.l, 0x7c
-; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7e, v1.l, s0
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v4.h, v4.l, s1
-; GFX1200-NEXT: v_or_b16 v4.l, v7.l, 0x7c
-; GFX1200-NEXT: s_and_b32 s1, s3, s2
+; GFX1200-NEXT: s_or_b32 s3, s4, s0
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v5.h
+; GFX1200-NEXT: v_or_b16 v6.l, v7.h, v8.l
+; GFX1200-NEXT: v_cmp_eq_u16_e64 s1, 30, v3.l
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 30, v3.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1200-NEXT: v_or_b16 v4.h, v6.l, v5.h
+; GFX1200-NEXT: v_or_b16 v5.h, v5.l, 0x7c
+; GFX1200-NEXT: s_and_b32 s0, s1, s0
+; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v1.h, v1.h
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, v5.l, s5
-; GFX1200-NEXT: s_or_b32 s2, s4, s1
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v0.l
+; GFX1200-NEXT: s_or_b32 s0, s2, s0
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v4.h, v3.h, vcc_lo
+; GFX1200-NEXT: v_or_b16 v3.h, v7.h, 0x7c
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v5.h, s3
+; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v0.l
+; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7e, v1.l, s1
+; GFX1200-NEXT: v_cmp_class_f16_e64 s2, v0.l, 0x204
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v4.l, s2
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s2, 0, v0.h
-; GFX1200-NEXT: v_cmp_class_f16_e64 s0, v0.l, 0x204
-; GFX1200-NEXT: v_and_b16 v2.h, v2.h, 0xff
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, v6.l, s1
-; GFX1200-NEXT: v_cmp_class_f16_e64 s1, v0.h, 0x204
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v7.l, s2
-; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v3.h, s0
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v0.h
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v1.h, v2.h, v5.l, vcc_lo
; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v0.l
-; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, v5.l, s0
-; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v0.h, v0.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v0.l, v3.l, v4.l, s1
-; GFX1200-NEXT: v_or_b16 v4.l, v3.h, v2.l
-; GFX1200-NEXT: v_or_b16 v2.h, v2.h, v1.l
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v3.l, v7.h, s0
+; GFX1200-NEXT: v_cmp_class_f16_e64 s0, v0.h, 0x204
+; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, v5.h, s2
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1200-NEXT: v_cndmask_b16 v0.l, v2.h, v3.h, s0
+; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v0.h, v0.h
+; GFX1200-NEXT: v_lshlrev_b16 v0.h, 8, v1.l
+; GFX1200-NEXT: v_cvt_u32_u16_e32 v3, v4.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v0.h, 0x7e, v1.h, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7e, v1.h, vcc_lo
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b16 v0.l, 0x7e, v0.l, s0
-; GFX1200-NEXT: v_cvt_u32_u16_e32 v1, v4.l
-; GFX1200-NEXT: v_mov_b16_e32 v3.l, v2.h
+; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v0.h
+; GFX1200-NEXT: v_or_b32_e32 v6, 0x7e7e0000, v3
+; GFX1200-NEXT: v_and_b16 v0.h, v1.l, 0xff
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_and_b16 v0.h, v0.h, 0xff
-; GFX1200-NEXT: v_lshlrev_b16 v2.l, 8, v0.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_or_b32_e32 v6, 0x7e7e0000, v1
-; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_or_b16 v0.l, v0.h, v2.l
-; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v2
+; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v0.l
+; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX1200-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_or_b16 v0.l, v0.h, v1.l
; GFX1200-NEXT: v_lshrrev_b64 v[5:6], 24, v[5:6]
-; GFX1200-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX1200-NEXT: v_mov_b16_e32 v3.l, v5.l
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
@@ -3677,19 +3665,18 @@ define <5 x i8> @to_bf8_v5f16(<5 x half> %x) {
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf8_f16_e64 v3.h, v1
+; GFX1250-TRUE16-NEXT: v_cvt_pk_bf8_f16_e64 v3.l, v1
; GFX1250-TRUE16-NEXT: v_cvt_pk_bf8_f16_e64 v5.h, s0
; GFX1250-TRUE16-NEXT: v_cvt_pk_bf8_f16_e64 v5.l, v2
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf8_f16_e64 v3.l, v0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.h
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_dual_lshlrev_b32 v4, 16, v1 :: v_dual_lshrrev_b32 v1, 8, v3
+; GFX1250-TRUE16-NEXT: v_cvt_pk_bf8_f16_e64 v0.l, v0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
+; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1250-TRUE16-NEXT: v_lshrrev_b64 v[6:7], 24, v[4:5]
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.l
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll b/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll
index ef22d69decc48..49049c38d2fff 100644
--- a/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll
@@ -1845,53 +1845,45 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX11-SDAG-TRUE16-LABEL: v_max3_bf16_maximumnum_maximumnum__v_v_v_0:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v4
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
@@ -1947,60 +1939,52 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v4
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3
; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -2417,17 +2401,11 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.l, s2
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s1
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v8.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v7
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v8
@@ -2436,11 +2414,8 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
@@ -2453,11 +2428,8 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
@@ -2467,17 +2439,11 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v2.h, v1.l, s1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s2
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v6
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v7
@@ -2486,11 +2452,8 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
@@ -2602,17 +2565,11 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.l, s2
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s1
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v8.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v7
; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v8
@@ -2623,11 +2580,8 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
@@ -2641,11 +2595,8 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
@@ -2657,17 +2608,11 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v2.h, v1.l, s1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s2
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v6
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v7
@@ -2677,12 +2622,9 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
diff --git a/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll b/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll
index 78dcf7141c8fd..2d9c18594ca6f 100644
--- a/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll
@@ -69,19 +69,17 @@ define bfloat @v_test_fmed3_r_i_i_bf16_minimumnum_maximumnum(bfloat %a) #1 {
; GFX11-SDAG-TRUE16-LABEL: v_test_fmed3_r_i_i_bf16_minimumnum_maximumnum:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v1
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v0.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, 2.0, v1
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v0.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 4.0, v1
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4080, v0.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -203,23 +201,17 @@ define <2 x bfloat> @v_test_fmed3_r_i_i_v2bf16_minimumnum_maximumnum(<2 x bfloat
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0x4000, v0.h, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v0.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, 2.0, v2
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, 2.0, v3
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0x4000, v1.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v0.l, s0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 4.0, v2
; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 4.0, v3
diff --git a/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll b/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll
index faa94b19030c6..aacb9a0f84973 100644
--- a/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll
@@ -1847,53 +1847,45 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX11-SDAG-TRUE16-LABEL: v_min3_bf16_minimumnum_minimumnum__v_v_v_0:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v4
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
@@ -1949,60 +1941,52 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v4
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3
; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -2422,17 +2406,11 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.l, s2
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s1
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v8.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v7
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v8
@@ -2441,11 +2419,8 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
@@ -2458,11 +2433,8 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
@@ -2472,17 +2444,11 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v2.h, v1.l, s1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s2
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v6
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v7
@@ -2491,11 +2457,8 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
@@ -2607,17 +2570,11 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.l, s2
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s1
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v8.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v7
; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v8
@@ -2628,11 +2585,8 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
@@ -2646,11 +2600,8 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
@@ -2662,17 +2613,11 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v2.h, v1.l, s1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s2
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v6
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v7
@@ -2682,12 +2627,9 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
diff --git a/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll b/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
index 3fc05f4e01947..088735ade33e6 100644
--- a/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
@@ -2746,13 +2746,12 @@ define <4 x i8> @test_signed_v4f16_v4i8(<4 x half> %f) {
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, v1.l, 0xff
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 0xff
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v1.l, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v0.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -2807,13 +2806,12 @@ define <4 x i8> @test_signed_v4f16_v4i8(<4 x half> %f) {
; GFX12-TRUE16-NEXT: v_and_b16 v1.l, v1.l, 0xff
; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
; GFX12-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 0xff
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v0.h
-; GFX12-TRUE16-NEXT: v_or_b16 v2.h, v1.l, v1.h
-; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-TRUE16-NEXT: v_or_b16 v2.l, v1.l, v1.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v0.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4454,41 +4452,40 @@ define <8 x i8> @test_signed_v8f16_v8i8(<8 x half> %f) {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0xff80
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.l, v3.l
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.h, v3.h
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, v2.l
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, v1.l
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v1.h
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, v2.l
; GFX11-TRUE16-NEXT: v_med3_i16 v3.l, v3.l, v4.l, 0x7f
; GFX11-TRUE16-NEXT: v_med3_i16 v3.h, v3.h, v4.l, 0x7f
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.h, v2.h
+; GFX11-TRUE16-NEXT: v_med3_i16 v2.l, v2.l, v4.l, 0x7f
; GFX11-TRUE16-NEXT: v_med3_i16 v1.l, v1.l, v4.l, 0x7f
; GFX11-TRUE16-NEXT: v_med3_i16 v1.h, v1.h, v4.l, 0x7f
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.h, v2.h
; GFX11-TRUE16-NEXT: v_and_b16 v3.l, v3.l, 0xff
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v3.h
+; GFX11-TRUE16-NEXT: v_and_b16 v4.h, v2.l, 0xff
+; GFX11-TRUE16-NEXT: v_med3_i16 v2.l, v2.h, v4.l, 0x7f
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, v1.l, 0xff
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
-; GFX11-TRUE16-NEXT: v_med3_i16 v2.l, v2.l, v4.l, 0x7f
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v3.l, v3.h
-; GFX11-TRUE16-NEXT: v_med3_i16 v3.l, v2.h, v4.l, 0x7f
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v0.h
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, v2.l, 0xff
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v2.l
; GFX11-TRUE16-NEXT: v_med3_i16 v0.l, v0.l, v4.l, 0x7f
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
; GFX11-TRUE16-NEXT: v_med3_i16 v0.h, v0.h, v4.l, 0x7f
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.h
+; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v1.l, v1.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.h
+; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v4.h, v2.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 0xff
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v0.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[5:6]
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v6
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -4557,41 +4554,40 @@ define <8 x i8> @test_signed_v8f16_v8i8(<8 x half> %f) {
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0xff80
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.l, v3.l
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.h, v3.h
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, v2.l
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, v1.l
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v1.h
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, v2.l
; GFX12-TRUE16-NEXT: v_med3_i16 v3.l, v3.l, v4.l, 0x7f
; GFX12-TRUE16-NEXT: v_med3_i16 v3.h, v3.h, v4.l, 0x7f
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.h, v2.h
+; GFX12-TRUE16-NEXT: v_med3_i16 v2.l, v2.l, v4.l, 0x7f
; GFX12-TRUE16-NEXT: v_med3_i16 v1.l, v1.l, v4.l, 0x7f
; GFX12-TRUE16-NEXT: v_med3_i16 v1.h, v1.h, v4.l, 0x7f
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.h, v2.h
; GFX12-TRUE16-NEXT: v_and_b16 v3.l, v3.l, 0xff
; GFX12-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v3.h
+; GFX12-TRUE16-NEXT: v_and_b16 v4.h, v2.l, 0xff
+; GFX12-TRUE16-NEXT: v_med3_i16 v2.l, v2.h, v4.l, 0x7f
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
; GFX12-TRUE16-NEXT: v_and_b16 v1.l, v1.l, 0xff
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v0.h
; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
-; GFX12-TRUE16-NEXT: v_med3_i16 v2.l, v2.l, v4.l, 0x7f
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
; GFX12-TRUE16-NEXT: v_or_b16 v6.h, v3.l, v3.h
-; GFX12-TRUE16-NEXT: v_med3_i16 v3.l, v2.h, v4.l, 0x7f
-; GFX12-TRUE16-NEXT: v_or_b16 v2.h, v1.l, v1.h
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v0.h
-; GFX12-TRUE16-NEXT: v_and_b16 v1.l, v2.l, 0xff
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v2.l
; GFX12-TRUE16-NEXT: v_med3_i16 v0.l, v0.l, v4.l, 0x7f
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v3.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
; GFX12-TRUE16-NEXT: v_med3_i16 v0.h, v0.h, v4.l, 0x7f
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.h
+; GFX12-TRUE16-NEXT: v_or_b16 v2.l, v1.l, v1.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.h
+; GFX12-TRUE16-NEXT: v_or_b16 v6.l, v4.h, v2.h
; GFX12-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 0xff
-; GFX12-TRUE16-NEXT: v_or_b16 v6.l, v1.l, v1.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v0.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v0.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
+; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[5:6]
-; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v6
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -6917,21 +6913,20 @@ define <4 x i8> @test_signed_v4f32_v4i8(<4 x float> %f) {
; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v2
; GFX11-TRUE16-NEXT: s_movk_i32 s0, 0xff80
; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v1, v1
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v4, v0
; GFX11-TRUE16-NEXT: v_med3_i32 v3, v3, s0, 0x7f
-; GFX11-TRUE16-NEXT: v_med3_i32 v4, v2, s0, 0x7f
+; GFX11-TRUE16-NEXT: v_med3_i32 v2, v2, s0, 0x7f
; GFX11-TRUE16-NEXT: v_med3_i32 v1, v1, s0, 0x7f
-; GFX11-TRUE16-NEXT: v_med3_i32 v0, v0, s0, 0x7f
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v3.l
-; GFX11-TRUE16-NEXT: v_and_b16 v2.h, v4.l, 0xff
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 0xff
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v3.l
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v2.l, 0xff
+; GFX11-TRUE16-NEXT: v_med3_i32 v3, v4, s0, 0x7f
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v3.l, 0xff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6976,22 +6971,21 @@ define <4 x i8> @test_signed_v4f32_v4i8(<4 x float> %f) {
; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v2
; GFX12-TRUE16-NEXT: s_movk_i32 s0, 0xff80
; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v1, v1
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v4, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_med3_i32 v3, v3, s0, 0x7f
-; GFX12-TRUE16-NEXT: v_med3_i32 v4, v2, s0, 0x7f
+; GFX12-TRUE16-NEXT: v_med3_i32 v2, v2, s0, 0x7f
; GFX12-TRUE16-NEXT: v_med3_i32 v1, v1, s0, 0x7f
-; GFX12-TRUE16-NEXT: v_med3_i32 v0, v0, s0, 0x7f
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v3.l
-; GFX12-TRUE16-NEXT: v_and_b16 v2.h, v4.l, 0xff
-; GFX12-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 0xff
-; GFX12-TRUE16-NEXT: v_or_b16 v2.h, v2.h, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v1.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
-; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v3.l
+; GFX12-TRUE16-NEXT: v_and_b16 v0.h, v2.l, 0xff
+; GFX12-TRUE16-NEXT: v_med3_i32 v3, v4, s0, 0x7f
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX12-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_and_b16 v0.l, v3.l, 0xff
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
index ac63bb67c7adf..25eaa0ac29758 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
@@ -2535,13 +2535,12 @@ define <4 x i8> @test_unsigned_v4f16_v4i8(<4 x half> %f) {
; GFX11-TRUE16-NEXT: v_min_u16 v0.h, v0.h, 0xff
; GFX11-TRUE16-NEXT: v_min_u16 v0.l, v0.l, 0xff
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v1.l, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v0.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -2589,13 +2588,12 @@ define <4 x i8> @test_unsigned_v4f16_v4i8(<4 x half> %f) {
; GFX12-TRUE16-NEXT: v_min_u16 v0.h, v0.h, 0xff
; GFX12-TRUE16-NEXT: v_min_u16 v0.l, v0.l, 0xff
; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v0.h
-; GFX12-TRUE16-NEXT: v_or_b16 v2.h, v1.l, v1.h
-; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-TRUE16-NEXT: v_or_b16 v2.l, v1.l, v1.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v0.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4143,41 +4141,40 @@ define <8 x i8> @test_unsigned_v8f16_v8i8(<8 x half> %f) {
; GFX11-TRUE16-LABEL: test_unsigned_v8f16_v8i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v1.h
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.h, v3.h
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v1.l
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v1.h
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, v3.l
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, v2.l
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.h, v2.h
-; GFX11-TRUE16-NEXT: v_min_u16 v1.h, v1.h, 0xff
; GFX11-TRUE16-NEXT: v_min_u16 v3.h, v3.h, 0xff
-; GFX11-TRUE16-NEXT: v_min_u16 v1.l, v1.l, 0xff
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v1.l
+; GFX11-TRUE16-NEXT: v_min_u16 v1.h, v1.h, 0xff
; GFX11-TRUE16-NEXT: v_min_u16 v3.l, v3.l, 0xff
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v3.h
-; GFX11-TRUE16-NEXT: v_min_u16 v4.l, v2.h, 0xff
+; GFX11-TRUE16-NEXT: v_min_u16 v4.l, v2.l, 0xff
+; GFX11-TRUE16-NEXT: v_min_u16 v2.l, v2.h, 0xff
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.h
+; GFX11-TRUE16-NEXT: v_min_u16 v1.l, v1.l, 0xff
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v0.h
-; GFX11-TRUE16-NEXT: v_min_u16 v4.h, v2.l, 0xff
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v3.l, v3.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v4.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v2.l
+; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v3.l, v2.h
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, v0.l
; GFX11-TRUE16-NEXT: v_min_u16 v0.h, v0.h, 0xff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.h
-; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v4.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v1.l, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v4.l, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.h
; GFX11-TRUE16-NEXT: v_min_u16 v0.l, v0.l, 0xff
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v8
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[7:8]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v8.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v0.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[5:6]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_unsigned_v8f16_v8i8:
@@ -4235,41 +4232,40 @@ define <8 x i8> @test_unsigned_v8f16_v8i8(<8 x half> %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v1.h
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.h, v3.h
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v1.l
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v1.h
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, v3.l
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, v2.l
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.h, v2.h
-; GFX12-TRUE16-NEXT: v_min_u16 v1.h, v1.h, 0xff
; GFX12-TRUE16-NEXT: v_min_u16 v3.h, v3.h, 0xff
-; GFX12-TRUE16-NEXT: v_min_u16 v1.l, v1.l, 0xff
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v1.l
+; GFX12-TRUE16-NEXT: v_min_u16 v1.h, v1.h, 0xff
; GFX12-TRUE16-NEXT: v_min_u16 v3.l, v3.l, 0xff
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v3.h
-; GFX12-TRUE16-NEXT: v_min_u16 v4.l, v2.h, 0xff
+; GFX12-TRUE16-NEXT: v_min_u16 v4.l, v2.l, 0xff
+; GFX12-TRUE16-NEXT: v_min_u16 v2.l, v2.h, 0xff
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.h
+; GFX12-TRUE16-NEXT: v_min_u16 v1.l, v1.l, 0xff
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v0.h
-; GFX12-TRUE16-NEXT: v_min_u16 v4.h, v2.l, 0xff
-; GFX12-TRUE16-NEXT: v_or_b16 v2.h, v1.l, v1.h
-; GFX12-TRUE16-NEXT: v_or_b16 v8.h, v3.l, v3.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v4.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v2.l
+; GFX12-TRUE16-NEXT: v_or_b16 v6.h, v3.l, v2.h
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, v0.l
; GFX12-TRUE16-NEXT: v_min_u16 v0.h, v0.h, 0xff
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.h
-; GFX12-TRUE16-NEXT: v_or_b16 v8.l, v4.h, v1.l
+; GFX12-TRUE16-NEXT: v_or_b16 v2.l, v1.l, v1.h
+; GFX12-TRUE16-NEXT: v_or_b16 v6.l, v4.l, v3.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.h
; GFX12-TRUE16-NEXT: v_min_u16 v0.l, v0.l, 0xff
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v0.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v8
-; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[7:8]
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v8.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v0.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
+; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[5:6]
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v7
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_unsigned_v8f16_v8i8:
@@ -6440,21 +6436,20 @@ define <4 x i8> @test_unsigned_v4f32_v4i8(<4 x float> %f) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v4, v2
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v4, v1
; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX11-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v3
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v2
; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.l
; GFX11-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v4
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6492,21 +6487,20 @@ define <4 x i8> @test_unsigned_v4f32_v4i8(<4 x float> %f) {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v4, v2
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v4, v1
; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX12-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v3
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v2
; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v3.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.l
; GFX12-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v4
-; GFX12-TRUE16-NEXT: v_or_b16 v2.h, v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v1.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
-; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v1.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
index 3523727664f03..9742ef219782f 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
@@ -13717,58 +13717,113 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX12-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB54_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB54_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB54_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -13813,54 +13868,105 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB54_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB54_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB54_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -14154,61 +14260,118 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX12-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB55_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB55_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB55_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB55_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -14255,56 +14418,108 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB55_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB55_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB55_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB55_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -14607,61 +14822,118 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX12-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB56_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB56_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB56_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB56_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -14709,56 +14981,108 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB56_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB56_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB56_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB56_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -15058,56 +15382,109 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX12-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB57_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: v_add_f32_e32 v3, v3, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB57_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v3
+; GFX12-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB57_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB57_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -15151,52 +15528,101 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB57_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_add_f32_e32 v3, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB57_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB57_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB57_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -17305,62 +17731,120 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX12-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB62_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB62_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB62_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB62_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB62_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB62_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -17407,56 +17891,108 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB62_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB62_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB62_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB62_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB62_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB62_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
index 11a808a040f16..43d041722da90 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
@@ -8614,58 +8614,113 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
; --------------------------------------------------------------------
define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB36_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -8710,54 +8765,105 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB36_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -9009,61 +9115,118 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(
}
define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB37_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -9110,56 +9273,108 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB37_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -9419,61 +9634,118 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_
}
define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB38_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -9521,56 +9793,108 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB38_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -9830,56 +10154,109 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_
}
define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: v_max_num_f32_e32 v3, v3, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB39_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v3
+; GFX12-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -9923,52 +10300,101 @@ define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_max_f32_e32 v3, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB39_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -11744,62 +12170,120 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_
}
define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB44_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -11846,56 +12330,108 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB44_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
index 53f165a636a7f..954d8101f58e4 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
@@ -8614,58 +8614,113 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
; --------------------------------------------------------------------
define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB36_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -8710,54 +8765,105 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB36_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -9009,61 +9115,118 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(
}
define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB37_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -9110,56 +9273,108 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB37_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -9419,61 +9634,118 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_
}
define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB38_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -9521,56 +9793,108 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB38_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -9830,56 +10154,109 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_
}
define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: v_min_num_f32_e32 v3, v3, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB39_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v3
+; GFX12-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -9923,52 +10300,101 @@ define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_min_f32_e32 v3, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB39_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -11744,62 +12170,120 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_
}
define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB44_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -11846,56 +12330,108 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB44_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
index 56a2efc5a9987..ba63812ea55cf 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
@@ -9223,58 +9223,113 @@ define void @global_system_atomic_fsub_noret_f16__offset12b_pos(ptr addrspace(1)
; --------------------------------------------------------------------
define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fsub_ret_bf16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB32_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB32_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB32_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB32_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fsub_ret_bf16:
; GFX942: ; %bb.0:
@@ -9319,54 +9374,105 @@ define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fsub_ret_bf16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB32_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB32_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB32_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB32_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fsub_ret_bf16:
; GFX10: ; %bb.0:
@@ -9616,61 +9722,118 @@ define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %
}
define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB33_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB33_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB33_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB33_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
; GFX942: ; %bb.0:
@@ -9717,56 +9880,108 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB33_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB33_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB33_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB33_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
; GFX10: ; %bb.0:
@@ -10024,61 +10239,118 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1)
}
define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB34_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB34_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB34_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
; GFX942: ; %bb.0:
@@ -10126,56 +10398,108 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB34_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB34_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0xfffff800
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB34_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
; GFX10: ; %bb.0:
@@ -10433,56 +10757,109 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1)
}
define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fsub_noret_bf16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB35_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: v_sub_f32_e32 v3, v3, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB35_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fsub_noret_bf16:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v3
+; GFX12-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB35_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fsub_noret_bf16:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB35_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fsub_noret_bf16:
; GFX942: ; %bb.0:
@@ -10526,52 +10903,101 @@ define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fsub_noret_bf16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB35_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_sub_f32_e32 v3, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB35_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_noret_bf16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB35_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_noret_bf16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB35_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fsub_noret_bf16:
; GFX10: ; %bb.0:
@@ -12337,62 +12763,120 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr addr
}
define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB40_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB40_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB40_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB40_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
; GFX942: ; %bb.0:
@@ -12439,56 +12923,108 @@ define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB40_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB40_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB40_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, v0, 0x7fe
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB40_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll
index b431c3f6f3606..550f329d0f28a 100644
--- a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll
+++ b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll
@@ -853,47 +853,95 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_cc_half(half inre
}
define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_cc_bfloat(bfloat inreg %a, bfloat %b) {
- ; DAGISEL-GFX11-WF32-LABEL: name: amdgpu_cs_chain_cc_bfloat
- ; DAGISEL-GFX11-WF32: bb.0 (%ir-block.0):
- ; DAGISEL-GFX11-WF32-NEXT: liveins: $sgpr0, $vgpr8
- ; DAGISEL-GFX11-WF32-NEXT: {{ $}}
- ; DAGISEL-GFX11-WF32-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; DAGISEL-GFX11-WF32-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX11-WF32-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec
- ; DAGISEL-GFX11-WF32-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc
- ; DAGISEL-GFX11-WF32-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[S_LSHL_B32_]], 0, [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec
- ; DAGISEL-GFX11-WF32-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec
- ; DAGISEL-GFX11-WF32-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767
- ; DAGISEL-GFX11-WF32-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], [[S_MOV_B32_]], implicit $exec
- ; DAGISEL-GFX11-WF32-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304
- ; DAGISEL-GFX11-WF32-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_ADD_F32_e64_]], [[S_MOV_B32_1]], implicit $exec
- ; DAGISEL-GFX11-WF32-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec
- ; DAGISEL-GFX11-WF32-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_ADD3_U32_e64_]], 0, [[V_OR_B32_e64_]], [[V_CMP_U_F32_e64_]], implicit $exec
- ; DAGISEL-GFX11-WF32-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX11-WF32-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
- ; DAGISEL-GFX11-WF32-NEXT: FLAT_STORE_SHORT_D16_HI [[COPY2]], [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
- ; DAGISEL-GFX11-WF32-NEXT: S_ENDPGM 0
+ ; DAGISEL-GFX11-WF32-TRUE16-LABEL: name: amdgpu_cs_chain_cc_bfloat
+ ; DAGISEL-GFX11-WF32-TRUE16: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX11-WF32-TRUE16-NEXT: liveins: $sgpr0, $vgpr8
+ ; DAGISEL-GFX11-WF32-TRUE16-NEXT: {{ $}}
+ ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[REG_SEQUENCE]], implicit $exec
+ ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc
+ ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[S_LSHL_B32_]], 0, [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec
+ ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767
+ ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], [[S_MOV_B32_]], implicit $exec
+ ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304
+ ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_ADD_F32_e64_]], [[S_MOV_B32_1]], implicit $exec
+ ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_ADD3_U32_e64_]], 0, [[V_OR_B32_e64_]], [[V_CMP_U_F32_e64_]], implicit $exec
+ ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[DEF2:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF2]]
+ ; DAGISEL-GFX11-WF32-TRUE16-NEXT: FLAT_STORE_SHORT_D16_HI [[COPY2]], [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
+ ; DAGISEL-GFX11-WF32-TRUE16-NEXT: S_ENDPGM 0
;
- ; DAGISEL-GFX11-WF64-LABEL: name: amdgpu_cs_chain_cc_bfloat
- ; DAGISEL-GFX11-WF64: bb.0 (%ir-block.0):
- ; DAGISEL-GFX11-WF64-NEXT: liveins: $sgpr0, $vgpr8
- ; DAGISEL-GFX11-WF64-NEXT: {{ $}}
- ; DAGISEL-GFX11-WF64-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; DAGISEL-GFX11-WF64-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX11-WF64-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec
- ; DAGISEL-GFX11-WF64-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc
- ; DAGISEL-GFX11-WF64-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[S_LSHL_B32_]], 0, [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec
- ; DAGISEL-GFX11-WF64-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec
- ; DAGISEL-GFX11-WF64-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767
- ; DAGISEL-GFX11-WF64-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], [[S_MOV_B32_]], implicit $exec
- ; DAGISEL-GFX11-WF64-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304
- ; DAGISEL-GFX11-WF64-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_ADD_F32_e64_]], [[S_MOV_B32_1]], implicit $exec
- ; DAGISEL-GFX11-WF64-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_64_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec
- ; DAGISEL-GFX11-WF64-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_ADD3_U32_e64_]], 0, [[V_OR_B32_e64_]], [[V_CMP_U_F32_e64_]], implicit $exec
- ; DAGISEL-GFX11-WF64-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; DAGISEL-GFX11-WF64-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
- ; DAGISEL-GFX11-WF64-NEXT: FLAT_STORE_SHORT_D16_HI [[COPY2]], [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
- ; DAGISEL-GFX11-WF64-NEXT: S_ENDPGM 0
+ ; DAGISEL-GFX11-WF32-FAKE16-LABEL: name: amdgpu_cs_chain_cc_bfloat
+ ; DAGISEL-GFX11-WF32-FAKE16: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX11-WF32-FAKE16-NEXT: liveins: $sgpr0, $vgpr8
+ ; DAGISEL-GFX11-WF32-FAKE16-NEXT: {{ $}}
+ ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec
+ ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc
+ ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[S_LSHL_B32_]], 0, [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec
+ ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767
+ ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], [[S_MOV_B32_]], implicit $exec
+ ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304
+ ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_ADD_F32_e64_]], [[S_MOV_B32_1]], implicit $exec
+ ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_ADD3_U32_e64_]], 0, [[V_OR_B32_e64_]], [[V_CMP_U_F32_e64_]], implicit $exec
+ ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF32-FAKE16-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; DAGISEL-GFX11-WF32-FAKE16-NEXT: FLAT_STORE_SHORT_D16_HI [[COPY2]], [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
+ ; DAGISEL-GFX11-WF32-FAKE16-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX11-WF64-TRUE16-LABEL: name: amdgpu_cs_chain_cc_bfloat
+ ; DAGISEL-GFX11-WF64-TRUE16: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX11-WF64-TRUE16-NEXT: liveins: $sgpr0, $vgpr8
+ ; DAGISEL-GFX11-WF64-TRUE16-NEXT: {{ $}}
+ ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[REG_SEQUENCE]], implicit $exec
+ ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc
+ ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[S_LSHL_B32_]], 0, [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec
+ ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767
+ ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], [[S_MOV_B32_]], implicit $exec
+ ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304
+ ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_ADD_F32_e64_]], [[S_MOV_B32_1]], implicit $exec
+ ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_64_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_ADD3_U32_e64_]], 0, [[V_OR_B32_e64_]], [[V_CMP_U_F32_e64_]], implicit $exec
+ ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[DEF2:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF2]]
+ ; DAGISEL-GFX11-WF64-TRUE16-NEXT: FLAT_STORE_SHORT_D16_HI [[COPY2]], [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
+ ; DAGISEL-GFX11-WF64-TRUE16-NEXT: S_ENDPGM 0
+ ;
+ ; DAGISEL-GFX11-WF64-FAKE16-LABEL: name: amdgpu_cs_chain_cc_bfloat
+ ; DAGISEL-GFX11-WF64-FAKE16: bb.0 (%ir-block.0):
+ ; DAGISEL-GFX11-WF64-FAKE16-NEXT: liveins: $sgpr0, $vgpr8
+ ; DAGISEL-GFX11-WF64-FAKE16-NEXT: {{ $}}
+ ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY]], implicit $exec
+ ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], 16, implicit-def dead $scc
+ ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_ADD_F32_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e64 0, [[S_LSHL_B32_]], 0, [[V_LSHLREV_B32_e64_]], 0, 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_BFE_U32_e64_:%[0-9]+]]:vgpr_32 = V_BFE_U32_e64 [[V_ADD_F32_e64_]], 16, 1, implicit $exec
+ ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32767
+ ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_ADD3_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD3_U32_e64 [[V_BFE_U32_e64_]], [[V_ADD_F32_e64_]], [[S_MOV_B32_]], implicit $exec
+ ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 4194304
+ ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_ADD_F32_e64_]], [[S_MOV_B32_1]], implicit $exec
+ ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_CMP_U_F32_e64_:%[0-9]+]]:sreg_64_xexec = nofpexcept V_CMP_U_F32_e64 0, [[V_ADD_F32_e64_]], 0, [[V_ADD_F32_e64_]], 0, implicit $mode, implicit $exec
+ ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_ADD3_U32_e64_]], 0, [[V_OR_B32_e64_]], [[V_CMP_U_F32_e64_]], implicit $exec
+ ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
+ ; DAGISEL-GFX11-WF64-FAKE16-NEXT: [[COPY2:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; DAGISEL-GFX11-WF64-FAKE16-NEXT: FLAT_STORE_SHORT_D16_HI [[COPY2]], [[V_CNDMASK_B32_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s16) into `ptr poison`)
+ ; DAGISEL-GFX11-WF64-FAKE16-NEXT: S_ENDPGM 0
;
; DAGISEL-GFX10-WF32-LABEL: name: amdgpu_cs_chain_cc_bfloat
; DAGISEL-GFX10-WF32: bb.0 (%ir-block.0):
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll
index b32df010240b0..8f2e67a618673 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll
@@ -17,13 +17,13 @@ define bfloat @v_exp2_bf16(bfloat %in) {
; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0
; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
@@ -151,10 +151,10 @@ define bfloat @v_exp2_fabs_bf16(bfloat %in) {
; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0
; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
@@ -290,10 +290,10 @@ define bfloat @v_exp2_fneg_fabs_bf16(bfloat %in) {
; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0
; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
@@ -430,10 +430,10 @@ define bfloat @v_exp2_fneg_bf16(bfloat %in) {
; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0
; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
@@ -565,13 +565,13 @@ define bfloat @v_exp2_bf16_fast(bfloat %in) {
; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0
; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
@@ -910,21 +910,20 @@ define <2 x bfloat> @v_exp2_fabs_v2bf16(<2 x bfloat> %in) {
; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0xc2fc0000, v1
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v5, 0, 0x42800000, s0
-; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 0xffffffc0, s0
-; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v1, v1, v5
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0x42800000, s0
+; GFX1200-SDAG-TRUE16-NEXT: v_dual_add_f32 v1, v1, v3 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0xffffffc0, s0
; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v1, v1
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0xffffffc0, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v3
-; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v1, v1, v4
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v1, v1, v3
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0
; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
@@ -1153,21 +1152,20 @@ define <2 x bfloat> @v_exp2_fneg_fabs_v2bf16(<2 x bfloat> %in) {
; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0xc2fc0000, v1
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v5, 0, 0x42800000, s0
-; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 0xffffffc0, s0
-; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v1, v1, v5
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0x42800000, s0
+; GFX1200-SDAG-TRUE16-NEXT: v_dual_add_f32 v1, v1, v3 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0xffffffc0, s0
; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v1, v1
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0xffffffc0, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v3
-; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v1, v1, v4
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v1, v1, v3
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0
; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
@@ -1396,21 +1394,20 @@ define <2 x bfloat> @v_exp2_fneg_v2bf16(<2 x bfloat> %in) {
; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0xc2fc0000, v1
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v5, 0, 0x42800000, s0
-; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 0xffffffc0, s0
-; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v1, v1, v5
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0x42800000, s0
+; GFX1200-SDAG-TRUE16-NEXT: v_dual_add_f32 v1, v1, v3 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0xffffffc0, s0
; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v1, v1
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0xffffffc0, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v3
-; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v1, v1, v4
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v1, v1, v3
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0
; GFX1200-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX1200-SDAG-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
index 7f3fd9c5a3364..af0e30394494f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
@@ -3822,20 +3822,35 @@ define float @v_log2_f32_from_fpext_bf16(bfloat %src) {
; GFX900-GISEL-NEXT: v_sub_f32_e32 v0, v0, v1
; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1100-SDAG-LABEL: v_log2_f32_from_fpext_bf16:
-; GFX1100-SDAG: ; %bb.0:
-; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, 32, vcc_lo
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
-; GFX1100-SDAG-NEXT: v_ldexp_f32 v0, v0, v2
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1100-SDAG-NEXT: v_log_f32_e32 v0, v0
-; GFX1100-SDAG-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-SDAG-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+; GFX1100-SDAG-TRUE16-LABEL: v_log2_f32_from_fpext_bf16:
+; GFX1100-SDAG-TRUE16: ; %bb.0:
+; GFX1100-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
+; GFX1100-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo
+; GFX1100-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX1100-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
+; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1100-SDAG-TRUE16-NEXT: v_log_f32_e32 v0, v0
+; GFX1100-SDAG-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX1100-SDAG-TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX1100-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-FAKE16-LABEL: v_log2_f32_from_fpext_bf16:
+; GFX1100-SDAG-FAKE16: ; %bb.0:
+; GFX1100-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
+; GFX1100-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 32, vcc_lo
+; GFX1100-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
+; GFX1100-SDAG-FAKE16-NEXT: v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-SDAG-FAKE16-NEXT: v_log_f32_e32 v0, v0
+; GFX1100-SDAG-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX1100-SDAG-FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX1100-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-GISEL-LABEL: v_log2_f32_from_fpext_bf16:
; GFX1100-GISEL: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
index 0377ad6dc3492..f0b4a91a2f11d 100644
--- a/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
@@ -107,27 +107,24 @@ define bfloat @v_maximumnum_bf16(bfloat %x, bfloat %y) #1 {
; GFX11-TRUE16-LABEL: v_maximumnum_bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
@@ -165,30 +162,28 @@ define bfloat @v_maximumnum_bf16(bfloat %x, bfloat %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -298,18 +293,19 @@ define bfloat @v_maximumnum_bf16_nnan(bfloat %x, bfloat %y) #1 {
; GFX11-TRUE16-LABEL: v_maximumnum_bf16_nnan:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -336,18 +332,20 @@ define bfloat @v_maximumnum_bf16_nnan(bfloat %x, bfloat %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -569,17 +567,11 @@ define <2 x bfloat> @v_maximumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) #1 {
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v7
@@ -588,11 +580,8 @@ define <2 x bfloat> @v_maximumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) #1 {
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
@@ -669,17 +658,11 @@ define <2 x bfloat> @v_maximumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) #1 {
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v6
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v7
@@ -690,11 +673,8 @@ define <2 x bfloat> @v_maximumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) #1 {
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
@@ -893,12 +873,9 @@ define <2 x bfloat> @v_maximumnum_v2bf16_nnan(<2 x bfloat> %x, <2 x bfloat> %y)
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v4
@@ -956,12 +933,9 @@ define <2 x bfloat> @v_maximumnum_v2bf16_nnan(<2 x bfloat> %x, <2 x bfloat> %y)
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v4
@@ -1264,61 +1238,51 @@ define <3 x bfloat> @v_maximumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) #1 {
; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v8, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v7, v7
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v4.l, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v4.l, s3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v9
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v10, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v9
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v8, v11
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v6.l, v4.l, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, s0
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
; GFX11-TRUE16-NEXT: s_and_b32 s0, s3, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
; GFX11-TRUE16-NEXT: s_and_b32 s1, s4, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s0
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v3bf16:
@@ -1393,66 +1357,55 @@ define <3 x bfloat> @v_maximumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) #1 {
; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v8, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v7, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, s0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s3
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v4.l, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v4.l, s3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v3.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v8
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v9
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v10, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v9
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v10
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v8, v11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v6.l, v4.l, s0
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
; GFX12-TRUE16-NEXT: s_and_b32 s0, s3, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
; GFX12-TRUE16-NEXT: s_and_b32 s1, s4, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s0
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v3bf16:
@@ -1711,19 +1664,14 @@ define <3 x bfloat> @v_maximumnum_v3bf16_nnan(<3 x bfloat> %x, <3 x bfloat> %y)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v2.h, v0.h, s0
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.h
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v7
; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
; GFX11-TRUE16-NEXT: s_and_b32 s0, s3, s0
@@ -1793,19 +1741,14 @@ define <3 x bfloat> @v_maximumnum_v3bf16_nnan(<3 x bfloat> %x, <3 x bfloat> %y)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v2.h, v0.h, s0
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.h
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v7
; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
; GFX12-TRUE16-NEXT: s_and_b32 s0, s3, s0
@@ -2195,75 +2138,62 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) #1 {
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v1.h, v3.h, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v10, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v4, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v4.l, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.h, v5.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v6.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.h, v6.l, s3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v10, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v9, v12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v7
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v10
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v6.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v8, v13
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v13, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v6.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v5.l
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v8
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
; GFX11-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v4.l, s5
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v5.l, s5
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v6.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v6.l, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v4bf16:
@@ -2355,84 +2285,71 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) #1 {
; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v1.h, v3.h, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v10, v10
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v4, v4
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v4.l, s0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.h, v5.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.h, v6.l, s3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v6.l, s3
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v6.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v10, v8
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v9, v12
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v7
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v10
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v6.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v8, v13
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v13, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v9
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v6.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v5.l
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v8
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
; GFX12-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v4.l, s5
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v5.l, s5
; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v6.l, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v6.l, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v4bf16:
@@ -2758,41 +2675,36 @@ define <4 x bfloat> @v_maximumnum_v4bf16_nnan(<4 x bfloat> %x, <4 x bfloat> %y)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v0
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v6
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v9, v8
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v0.h
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v11, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.h, v1.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v3.h, v1.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v1.h
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v4
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v0.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v7
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v9
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v1.h, s0
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, s4
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s5, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v0.h, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v2.h, v0.h, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.h
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s0, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s4, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v6.l, v1.h, s5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v4bf16_nnan:
@@ -2861,45 +2773,41 @@ define <4 x bfloat> @v_maximumnum_v4bf16_nnan(<4 x bfloat> %x, <4 x bfloat> %y)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v0
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v6
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v9, v8
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v0.h
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v11, v10
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.h, v1.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v3.h, v1.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v1.h
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v0.h, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.h
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v1.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v0.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v7
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v8
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v9
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v2.h, v0.h, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.h
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
+; GFX12-TRUE16-NEXT: s_and_b32 s5, s0, s2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s3, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s4, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v1.h, s0
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s2
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s3, s4
-; GFX12-TRUE16-NEXT: s_and_b32 s2, s5, s6
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v6.l, v1.h, s5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, s1
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v0.h, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v4bf16_nnan:
@@ -3449,110 +3357,92 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) #1 {
; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v5.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v5.h, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v11, v11
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v12, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v5.h, v6.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v5.h, v7.l, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v13, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v10, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v16
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v11, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v1.h, v4.h, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v1.h, v4.h, s1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s5, v8, v9
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v14, v14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v4.h, v8.l, s2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v15, v15
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v16, v16
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v12, v12
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v15, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v16.l, v7.l, s5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v4.h, v6.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v7.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v14, v14
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s5, v9, v12
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v10.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v17, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v0.h, v3.h, s5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v15
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v0.h, v3.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s2, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v14, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v7.l, v6.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v4.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v3.h, v9.l, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v10.l, v8.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v9.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v8.l, v7.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v3.h, v9.l, s6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v13, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v12, v11
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v14, v13
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v16, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v12
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v16, v15
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v10
; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v9.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v8.l, v9.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.l
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v4.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v8.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v2.l
+; GFX11-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v11
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v9.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT: s_and_b32 s7, s0, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v11
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v1.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v12
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v0.l
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v7.l, v8.l, s7
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v10.l, v6.l, s7
; GFX11-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v9.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v9.l, s2
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, s3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v2.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v2.l, s1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v6bf16:
@@ -3687,122 +3577,104 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) #1 {
; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v4
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v5.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v5.h, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v11, v11
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v12, v12
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v5.h, v6.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v5.h, v7.l, s0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v13, v13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v10, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v16
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v11, v11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v1.h, v4.h, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v9
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v1.h, v4.h, s1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s5, v8, v9
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v14, v14
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v4.h, v8.l, s2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v15, v15
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v16, v16
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v8.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v10.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v17, v17
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v12, v12
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v15, v15
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v0.h, v3.h, s5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v15
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v16.l, v7.l, s5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v4.h, v6.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v7.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v8
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v14, v14
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s5, v9, v12
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v0.h, v3.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s2, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s4
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v2.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v14, v11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v7.l, v6.l, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v4.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v3.h, v9.l, s6
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s8
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v8.l, v7.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v3.h, v9.l, s6
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v13, v14
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v10.l, v8.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v9.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v11
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v1.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v6.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v3.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v0.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v10
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v12, v11
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v14, v13
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v16, v15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v12
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v16, v15
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v10
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v9.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v8.l, v9.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.l
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v11
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v4.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v6.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v8.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v2.l
+; GFX12-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v11
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v9.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX12-TRUE16-NEXT: s_and_b32 s7, s0, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v11
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v1.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v12
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v0.l
-; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v7.l, v8.l, s7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v10.l, v6.l, s7
; GFX12-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v9.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v9.l, s2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, s3
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v2.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v2.l, s1
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v6bf16:
@@ -4583,149 +4455,125 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) #1 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v3.h, v7.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v7.h, v8.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v3.h, v7.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v7.h, v9.l, s0
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v7.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v9
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v3.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v2.h, v6.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v7.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v6.h, v10.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v15, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v11.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v8.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v14, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v16
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v18
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v9.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v2.h, v6.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v9.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v6.h, v8.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v9.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v15, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v16
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v14, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v10
; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v3.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v14, v16
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v15, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v7.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v10.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v8.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v11.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v10.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v12, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v9.l, v8.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v1.h, v5.h, s2
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v11, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v6.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v16.l, v8.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, s2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v10.l, v9.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v1.h, v5.h, s4
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v3.l
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v5.h, v8.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v9.l
-; GFX11-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v5.h, v10.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v8.l
+; GFX11-TRUE16-NEXT: s_and_b32 s7, s0, s1
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v10
; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s3
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v13, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v11.l, v10.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v7.l, v3.l, s7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v12.l, v8.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v14, v14
; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v5
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v10, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v0.h, v4.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v12, v12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v1.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v4.h, v10.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v10.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v11, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v0.h, v4.h, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v13.l, v10.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v4.h, v11.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v0.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v8.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v11.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v4.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v6.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v14, v12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v6
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v14, v13
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v16, v15
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v18, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v12
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v18, v17
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v10.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v11.l, s0
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v5.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v4.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v8.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v10.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v8
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11-TRUE16-NEXT: s_and_b32 s8, s0, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v13
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v1.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v12
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v10.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4
+; GFX11-TRUE16-NEXT: s_and_b32 s8, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v13
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v11.l
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v14
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v0.l
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v9.l, v8.l, s8
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v12.l, v10.l, s8
; GFX11-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v11.l, v10.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v11.l, s2
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.l, v0.l, s3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v6.l, v2.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v7.l, v3.l, s7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v6.l, v2.l, s1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v8bf16:
@@ -4885,171 +4733,145 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v6
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v3.h, v7.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v3.h, v7.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v7.h, v8.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v7.h, v9.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v7.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v9
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v3.l, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v2.h, v6.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v11
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v7.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v2.h, v6.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v9.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v6.h, v10.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v15, v12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v16
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v10.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v11.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v6.h, v8.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v8.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v14, v12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v16
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v18
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v9.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v9.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v15, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v16
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v14, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v10
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v3.l, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v14, v16
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v15, v15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v18
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v7.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v11, v15
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v10.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v8.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v14
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v11.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v10.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v12, v12
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v6.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v16.l, v8.l, s1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s3, s2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v9.l, v8.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v14
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v1.h, v5.h, s2
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v6.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v10.l, v9.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v12
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v1.h, v5.h, s4
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v3.l
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v9
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v5.h, v8.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v8.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v9.l
-; GFX12-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v5.h, v10.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v8.l
+; GFX12-TRUE16-NEXT: s_and_b32 s7, s0, s1
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v10
; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s3
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v13, v12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v11.l, v10.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v15
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v7.l, v3.l, s7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v12.l, v8.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v14, v14
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v5
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v10, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v0.h, v4.h, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v12, v12
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v1.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v11, v11
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v0.h, v4.h, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, s2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v4.h, v10.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v10.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v13.l, v10.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v4.h, v11.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v0.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v8.l, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v11.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v4.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v6.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v14, v12
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v6
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v14, v13
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v16, v15
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v18, v17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v12
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v18, v17
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v10.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v11.l, s0
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v5.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v12
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v11.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v4.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v8.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v10.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v8
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX12-TRUE16-NEXT: s_and_b32 s8, s0, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v13
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v1.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v12
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v10.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4
+; GFX12-TRUE16-NEXT: s_and_b32 s8, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v13
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v11.l
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v14
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v0.l
-; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v9.l, v8.l, s8
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v12.l, v10.l, s8
; GFX12-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v11.l, v10.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v11.l, s2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.l, v0.l, s3
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v1.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v6.l, v2.l, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v7.l, v3.l, s7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v6.l, v2.l, s1
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v8bf16:
@@ -6466,303 +6288,250 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) #
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v7
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v15
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v24, v24
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v7.h, v15.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v7.h, v15.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v15
; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v15.l, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v14.l, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v16.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v15.h, v16.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v16.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v17.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v7.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v15.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v6.h, v14.h, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v20, v21
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v14.h, v18.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v16.l, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v21, v20
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v18.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v18
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v15.h, v18.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v18.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v14.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v17
; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v7.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v14
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v7.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v20, v19
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v6.h, v14.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v17.l, v18.l, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v21
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v14.h, v21.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v20, v19
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v15.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v21, v20
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v17
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v7.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v19
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v22
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v6.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v18.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v17.l, v16.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v14
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v20
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v19.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v16, v16
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v17
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v15.l, v7.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v5.h, v13.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v14.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v16
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v15.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v13.h, v15.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v13
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s0, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v16.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v21, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v13.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v23
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v22, v20
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v15
+; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v21.l, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v20, v20
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v16.l, v18.l, s2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v19
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v14
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s3, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v18, v18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v20, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v5.h, v13.h, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v13.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v18, v18
; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s2
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v21, v20
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v13.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v4.h, v12.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v14.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v20
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v15.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v12.h, v17.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v15.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v21, v23
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v16.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v22
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v24
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v21.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v13.h, v19.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v15.l, v7.l, s4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v13
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v24, v24
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v20, v18
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v17.l, v21.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v15
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v23, v22
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v19.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v19.l, v18.l, s4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s3, v23, v21
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v22
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v13.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v4.h, v12.h, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v15, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v16
; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v20.l, v17.l, s3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v22
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13
; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v14.l, v6.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v16.l, v15.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v18.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v12.h, v17.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v15
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v19.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s1, s2
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v18, v15
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v20, v20
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v12
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v19
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v5.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v17.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v12.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v15, v15
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v16, v16
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v3.h, v11.h, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v16.l, v19.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v13.l, v5.l, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v11.h, v15.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v13
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v16, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v4
+; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v20, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v11.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v21, v18
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v14.l, v17.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v15.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v2.h, v10.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v12
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v16
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v18, v17
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v15.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v10.h, v19.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v3.h, v11.h, s2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v15, v15
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v17.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v13.l, v5.l, s3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v4.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v11.h, v14.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v3
-; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v14.l
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v11
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v16, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v20
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v21
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v11.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v18.l, v17.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v16, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v12.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v14.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v3.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v2.h, v10.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v13.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v10.h, v15.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v20
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v19, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v15.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v17
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v14.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v11.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v18, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v13.l, v14.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v19, v19
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v4.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v19
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v2
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v13.l, v15.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v11
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v17, v14
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v20
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v16.l, v15.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v16.l, v19.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v10.l, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v17
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v13.l
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v16, v16
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v15, v15
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v13
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v19.l
; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v1.h, v9.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v16, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v10.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v17
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v15.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v9.h, v14.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v14.l
-; GFX11-TRUE16-NEXT: s_and_b32 s8, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v16.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s3
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v18, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v13.l, v15.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v20
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v1.h, v9.h, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v9.h, v16.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v15, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v13.l, v19.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v15, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v18, v18
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v13, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v0.h, v8.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v8.l, s2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v17, v17
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v1.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v8.h, v13.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v13.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v15, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s1, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v9
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v15, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v0.h, v8.h, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v8.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v8.h, v15.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v15
; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v0.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v14.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v15.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v8.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v10.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v19, v17
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v16.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v16.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v19, v18
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v21, v20
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v23, v22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v23, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v17
; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v13.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v14.l, v15.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v16.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s2
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v9.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v17
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v8.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v14.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; GFX11-TRUE16-NEXT: s_and_b32 s9, s0, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v18
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v1.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v17
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v13.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v8
+; GFX11-TRUE16-NEXT: s_and_b32 s9, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v14
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v18
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v15.l
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v19
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v0.l
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v16.l, v14.l, s9
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v17.l, v16.l, s9
; GFX11-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v15.l, v13.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v13.l, v15.l, s2
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, s3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v9.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v10.l, v2.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v10.l, v2.l, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v11.l, v3.l, s8
; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v12.l, v4.l, s7
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -7089,346 +6858,291 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) #
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v7
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v7
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v15
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v24, v24
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v7.h, v15.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v7.h, v15.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v15
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v15.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v14.l, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v16.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v18
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v15.h, v16.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v16.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v17.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v15.h, v18.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v18.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v14.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v17
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v7.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v18
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v15.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v6.h, v14.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v20, v21
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v22
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v23
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v14
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v7.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v20, v19
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v15
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v14.h, v18.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v6.h, v14.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v16.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v21, v20
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v19.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v18.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v17.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v17.l, v18.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v21
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v7.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v15.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v21, v20
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v22
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v6.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v23
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v18.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v17.l, v16.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v14
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v20
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v7.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v19.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v16, v16
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v17
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v13
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v14.h, v21.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v20, v19
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v16
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v17
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v7.l, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v19
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v22, v20
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v14
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v15
+; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v21.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v20, v20
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v19
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v15.l, v7.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v5
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v5.h, v13.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v14.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v16
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v18.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v15.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v13.h, v15.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v13
-; GFX12-TRUE16-NEXT: s_and_b32 s4, s0, s1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v16.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v21, v17
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v13.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v22
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v23
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v12
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v16.l, v18.l, s2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v17
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v6
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v19
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v14
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v13
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s3, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v18, v18
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v20, v19
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v5.h, v13.h, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v13.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v18, v18
; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s2
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v21, v20
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v5.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v13.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v4.h, v12.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v14.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v20
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v21.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v13.h, v19.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v19
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v15.l, s2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v12.h, v17.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v15.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v21, v23
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v17.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v20.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v16.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v22
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v15.l, v7.l, s4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v13
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v24, v24
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v20, v18
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v17.l, v21.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v15
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v23, v22
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v12
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v24
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v19.l, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v19.l, v18.l, s4
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s3, v23, v21
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v22
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v13.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v4.h, v12.h, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v15, v15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v16
; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v20.l, v17.l, s3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v22
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v5.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v14.l, v6.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v16.l, v15.l, s1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v18.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v12.h, v17.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v15
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v19.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v18
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s1, s2
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v18, v15
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v20, v20
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v12
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v19
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v5.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v11
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v17.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v12.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v15, v15
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v16, v16
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v11
-; GFX12-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v14
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v3.h, v11.h, s2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v15, v15
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v17.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v3.h, v11.h, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v14
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v13.l, v5.l, s3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v12.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v4.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v16.l, v19.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v13.l, v5.l, s2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v11.h, v14.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v3
-; GFX12-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v13.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v14.l
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v11
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v16, v15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v20
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v21
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v11.h, v15.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v11.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v17.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v13
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v4
+; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v20, v19
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v11.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v21, v18
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v18.l, v17.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v16, v15
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v14.l, v17.l, s2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v10
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v12.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v15.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v14.l, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v11.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v3.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v2.h, v10.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v16
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v13.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v2.h, v10.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v12
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v16
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v18, v17
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v15.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v10.h, v15.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v20
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v19, v18
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v16.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v15.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v17
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v14.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v10.h, v19.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v14
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v4.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v19
-; GFX12-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v11.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v18, v17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, s2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v19
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v2
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX12-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v13.l, v14.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v19, v19
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v13.l, v15.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v11
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v17, v14
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v20
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v16.l, v15.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v16.l, v19.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v10.l, s0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v17
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v13.l
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v16, v16
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v9
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v15, v15
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v13
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v19.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v14
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v1.h, v9.h, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v16, v16
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v10.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v17
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v15.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v9.h, v14.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v19
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v14.l
-; GFX12-TRUE16-NEXT: s_and_b32 s8, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v16.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s3
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v18, v17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v1.h, v9.h, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v9.h, v16.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v15, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v17
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v13.l, v15.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v19
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v20
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v13.l, v19.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v9
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v15, v13
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v18, v18
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v13, v13
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v15, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s8, s1, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v9
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v8
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v8
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v15, v15
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v0.h, v8.h, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v8.l, s2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v17, v17
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v9.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v1.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v8.h, v13.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v13.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v0.h, v8.h, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v8.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, s2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v8.h, v15.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v15
; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v0.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v14.l, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v15.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v8.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v10.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v19, v17
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v16.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v8
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v16.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v19, v18
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v21, v20
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v23, v22
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v23, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v17
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v13.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v14.l, v15.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v16.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s2
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v9.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v17
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v15.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v8.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v14.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; GFX12-TRUE16-NEXT: s_and_b32 s9, s0, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v18
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v1.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v17
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v13.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v8
+; GFX12-TRUE16-NEXT: s_and_b32 s9, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v14
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v18
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v15.l
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v19
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v0.l
-; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v16.l, v14.l, s9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v17.l, v16.l, s9
; GFX12-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v15.l, v13.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v13.l, v15.l, s2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, s3
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v9.l, v1.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v10.l, v2.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v10.l, v2.l, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v11.l, v3.l, s8
; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v12.l, v4.l, s7
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -10256,594 +9970,492 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v36, 16, v14
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v13
; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v30
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v15
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v36, v36
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v29
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v38, v38
; GFX11-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v15
; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v30
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v34, v34
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v37, v37
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v30.l, s4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v33, v33
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v39, v39
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v13.h, v29.h, s6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v36, 16, v14
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v34, v34
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v29
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v12
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v28
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v35, v35
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v48, v48
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v14.h, v30.h, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v30
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v36, v36
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v12
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v27
; GFX11-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v26
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v114, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v49, v49
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v50, v50
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v51, v51
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v54, v54
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v68, v68
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v29.l, s7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.l, v30.h, v32.l, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v11
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v37, v37
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v30.l, s3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v29
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v27
; GFX11-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v32, v32
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v35, v35
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v14.h, v30.h, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v22
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v50, v50
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v50.l, v29.h, v33.l, s7
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v28
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v99, 0xffff0000, v22
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v49, v49
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v66, v66
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v67, v67
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s46, v100, v100
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.l, v30.h, v32.l, s3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v100.l, v30.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v9
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v52, v52
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v55, v55
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v66, v66
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v69, v69
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s59, v114, v114
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v12.h, v28.h, s9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v11.h, v27.h, s13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v26.l, s19
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v32
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v51
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v28
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v48, v48
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v52, v52
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v68, v68
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v101, v101
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v101.l, v50.l
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v51, v51
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s45, v99, v99
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v12.h, v28.h, s10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.l, v10.h, v26.h, s18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v32.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v99.l, v49.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v67, 16, v67
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v100
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v11
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v53, v53
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v29.l, s8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v28.l, s12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v68
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v101
; GFX11-TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.l, v28.h, v34.l, s11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v53.l, v26.h, v36.l, s19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v66, 16, v66
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v99
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s19, v67, v100
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v26
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v115, 0xffff0000, v20
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v64, v64
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v15
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v102, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v38, v38
; GFX11-TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v25
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v64, v64
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v26.l, s20
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s13
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s20, v68, v101
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v70, v70
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v34.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v112.l, v51.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s18, v66, v99
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s19
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v65, v65
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v69, v69
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v71, v71
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v27.l, s16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v103.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v113.l, v28.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v50.l, v50.l, v33.l, s20
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v70
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v112
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.l, v49.l, v32.l, s18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v30.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v9
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v25
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.l, v9.h, v25.h, s22
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v69
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v71, 16, v71
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v103
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v113
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v50.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s22, v70, v112
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v49.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v67, 16, v67
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v8
; GFX11-TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v24
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v54, v54
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v80, v80
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v81, v81
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.l, v11.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s21
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.l, v25.h, v37.l, s23
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v115.l, v27.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s21, v69, v103
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s23, v71, v113
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v68
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.l, v51.l, v34.l, s22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v66, 16, v66
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s19, 0, v67
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v82, v82
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v83, v83
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v25.l, s24
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.l, v10.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v81
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v117.l, v26.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v115
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s21
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s23
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s20, 0, v68
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v65, v65
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v67, v67
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v80, v80
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v82, v82
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.l, v10.h, v26.h, s17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v65.l, v28.h, v34.l, s10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v68.l, v27.h, v35.l, s14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s20
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v29
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s20, v52, v114
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v53, v53
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v70, v70
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s60, v115, v115
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v27.l, s15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v30
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v103, 0xffff0000, v21
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v33, v33
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v39, v39
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s47, v102, v102
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v13.h, v29.h, s5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v24
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v23
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v84, v84
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v96, v96
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v51.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s18, 0, v66
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.l, v8.h, v24.h, s26
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v83
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v117
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s25, v81, v115
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v28.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v71, v71
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v81, v81
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v83, v83
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v84, v84
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v25.l, s23
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.l, v8.h, v24.h, s25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v66, 16, v34
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v35
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v71.l, v26.h, v36.l, s18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v65
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v68
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s23, v64, v117
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.l, v51.l, v32.l, s20
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.l, v9.h, v25.h, s21
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s16
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s21, v53, v115
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s56, v103, v103
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.l, v5.h, v21.h, s47
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.l, v29.h, v33.l, s6
; GFX11-TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v85, v85
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s43, v97, v97
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v24.l, s28
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v23.l, s42
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v70
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v98, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v55, v55
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v55.l, v24.h, v38.l, s27
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s27, v83, v117
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v69
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v71, 16, v71
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v85, v85
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v24.l, s27
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v36
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s24
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v85.l, v24.h, v38.l, s26
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v71
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s24, v66, v118
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s26, v69, v128
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v51
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v129, 16, v27
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v33
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v102.l, v21.h, v49.l, s56
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v54
; GFX11-TRUE16-NEXT: v_and_b32_e32 v87, 0xffff0000, v23
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.l, v9.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s29
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v7.l, s43
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.l, v25.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s22, 0, v70
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s44, v98, v98
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s27
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.l, v27.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s21, 0, v69
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s23, 0, v71
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v86, v86
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v87, v87
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.l, v7.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v85
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v129.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.l, v23.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v119, 16, v119
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v48.l, v6.h, v22.h, s44
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.l, v26.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v81
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.l, v7.h, v23.h, s40
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v87
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v97
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v129, 16, v129
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v131
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s29, v85, v119
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v14.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v65.l, v22.h, v48.l, s45
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0, v11.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v83
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s25, 0, v81
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s21, s5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v64.l, v23.h, v39.l, s41
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s41, v87, v129
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s43, v97, v131
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s29
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v29.l, v13.l, s5
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v12.l
-; GFX11-TRUE16-NEXT: s_and_b32 s3, s19, s3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v98.l, v48.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v132.l, v65.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0, v10.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s27, 0, v83
-; GFX11-TRUE16-NEXT: s_and_b32 s9, s25, s9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v30.l, v14.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s41
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v7.l, s43
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.l, v25.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v27.l, v11.l, s9
-; GFX11-TRUE16-NEXT: s_and_b32 s7, s23, s7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v98
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v132
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v11.h, v27.h, s14
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v32.l
-; GFX11-TRUE16-NEXT: s_and_b32 s11, s27, s11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v28.l, v12.l, s7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.l, v23.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v85
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v26.l, v10.l, s11
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s44, v98, v132
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.l, v27.h, v35.l, s15
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s18, s2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0, v9.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v87
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v97
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s29, 0, v85
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v49.l, v32.l, s2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v102, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v22.l, s46
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v65.l, v65.l, v48.l, s44
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0, v8.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0, v7.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s41, 0, v87
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s43, 0, v97
-; GFX11-TRUE16-NEXT: s_and_b32 s13, s29, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v98.l, v65.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v25.l, v9.l, s13
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v21
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s15, s41, s15
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v33.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v24.l, v8.l, s15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v98
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.l, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.l, v52.l
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s20, s4
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v35.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v24
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v50.l, v33.l, s4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v80
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v114
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v36.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v84.l, v37.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v38.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v39.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s24, v80, v114
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v116.l, v53.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v118.l, v54.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v128.l, v55.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v130.l, v64.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.l, v52.l, v35.l, s24
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v82
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v84
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.l, v52.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v116
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v118
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v128
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v130
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v80
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s26, v82, v116
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s28, v84, v118
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s40, v86, v128
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s42, v96, v130
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s24, 0, v80
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v53.l, v53.l, v36.l, s26
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.l, v54.l, v37.l, s28
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v55.l, v55.l, v38.l, s40
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v64.l, v64.l, v39.l, s42
-; GFX11-TRUE16-NEXT: s_and_b32 s8, s24, s8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v53.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v52.l, v35.l, s8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v84.l, v54.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v55.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v64.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v34.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v82
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v84
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96
-; GFX11-TRUE16-NEXT: s_and_b32 s6, s22, s6
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0, v36.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0, v37.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0, v38.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0, v39.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s26, 0, v82
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s28, 0, v84
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s40, 0, v86
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s42, 0, v96
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v51.l, v34.l, s6
-; GFX11-TRUE16-NEXT: s_and_b32 s10, s26, s10
-; GFX11-TRUE16-NEXT: s_and_b32 s12, s28, s12
-; GFX11-TRUE16-NEXT: s_and_b32 s14, s40, s14
-; GFX11-TRUE16-NEXT: s_and_b32 s16, s42, s16
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v53.l, v36.l, s10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v54.l, v37.l, s12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v55.l, v38.l, s14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v64.l, v39.l, s16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v23
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v98, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v22
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v86, v86
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v96, v96
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s45, v100, v100
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v32.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s28
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v26
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s28, v80, v130
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v53.l, v65.l, v34.l, s24
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v29
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s20, 0, v69
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v5
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s27, v70, v129
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v30
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v82.l, v25.h, v37.l, s22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v49
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v102
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s22, v55, v116
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v87, v87
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v97, v97
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s43, v98, v98
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s46, v101, v101
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.l, v7.h, v23.h, s29
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v23.l, s41
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v22.l, s45
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v13.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v135, 16, v24
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s29, v81, v131
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v53
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s23, 0, v80
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s20, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v21
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s57, v112, v112
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s21, 0, v70
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v51.l, v32.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s56, v103, v52
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.l, v54.l, v33.l, s22
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v99, 0xffff0000, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v48.l, v6.h, v22.h, s43
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v14.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v34.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v38
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v7.l, s42
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, s46
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v134, 16, v85
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s43, v87, v135
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s29
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s24, 0, v81
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s23, s4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s58, v113, v113
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v28.l, s11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v21.l, s57
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v29.l, v13.l, s4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v96.l, v23.h, v39.l, s40
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v133, 16, v25
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v55.l, v71.l, v36.l, s28
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v71, 16, v52
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s44, v99, v99
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v145, 16, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v147, 16, v22
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s42, v86, v134
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s43
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v26
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s21, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s24, s5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v67, 16, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s58
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v30.l, v14.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v53.l, v34.l, s5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v33.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v39
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v144, 16, v96
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s41, v84, v133
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s27
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s22, 0, v71
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0, v10.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v99.l, v22.h, v48.l, s44
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s45, v98, v145
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s47, v101, v147
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v24
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s29, 0, v86
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v21
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s44, v97, v144
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s41
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v27
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s22, s3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0, v8.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v48
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v146, 16, v99
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v7.l, s45
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, s47
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s43, 0, v98
+; GFX11-TRUE16-NEXT: s_and_b32 s10, s29, s10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v52.l, v33.l, s3
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s57, v112, v114
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v26.l, v10.l, s10
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v11.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v66.l, v96.l, v39.l, s44
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v25
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s27, 0, v84
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s46, v100, v146
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.l, v68.l, v35.l, s26
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v68.l, v102.l, v49.l, s56
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v102, 16, v22
+; GFX11-TRUE16-NEXT: s_and_b32 s14, s43, s14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v50.l, v4.h, v20.h, s59
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s57
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v24.l, v8.l, s14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s12
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0, v9.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s41, 0, v96
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s27, s8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0, v7.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 0, v6.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s45, 0, v100
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s47, 0, v102
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v27.l, v11.l, s8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v113.l, v20.h, v50.l, s60
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v21
+; GFX11-TRUE16-NEXT: s_and_b32 s12, s41, s12
+; GFX11-TRUE16-NEXT: s_and_b32 s16, s45, s16
+; GFX11-TRUE16-NEXT: s_and_b32 s18, s47, s18
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v25.l, v9.l, s12
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s57, 0, v112
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v23.l, v7.l, s16
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v22.l, v6.l, s18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v113
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v50
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v119, 16, v28
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v37
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v82
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v68
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v12.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s25, v67, v119
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 0, v49.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s40, v83, v132
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s56, 0, v103
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v54
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s25
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v35.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v64.l, v82.l, v37.l, s40
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v65.l, v85.l, v38.l, s42
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s26, 0, v83
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v28
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v55
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v67.l, v99.l, v48.l, s46
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0, v36.l
+; GFX11-TRUE16-NEXT: s_and_b32 s7, s26, s7
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s25, 0, v82
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v54.l, v35.l, s7
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s28, 0, v85
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v64
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v65
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s25, s6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v66
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v28.l, v12.l, s6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v67
+; GFX11-TRUE16-NEXT: s_and_b32 s9, s28, s9
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0, v37.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0, v38.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0, v39.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0, v48.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s40, 0, v87
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s42, 0, v97
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s44, 0, v99
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s46, 0, v101
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v55.l, v36.l, s9
+; GFX11-TRUE16-NEXT: s_and_b32 s11, s40, s11
+; GFX11-TRUE16-NEXT: s_and_b32 s13, s42, s13
+; GFX11-TRUE16-NEXT: s_and_b32 s15, s44, s15
+; GFX11-TRUE16-NEXT: s_and_b32 s17, s46, s17
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v64.l, v37.l, s11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v65.l, v38.l, s13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v66.l, v39.l, s15
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v67.l, v48.l, s17
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v31
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v31
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v66.l, v15.h, v31.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v67, v67
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v68, v68
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v66.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v15.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v67.l, v31.h, v66.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.l, v31.l, v15.l, s1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v68
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v69
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v31
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v31
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v15.h, v31.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v67.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v31.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v70
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v71, 16, v71
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v68, v70
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v69, v71
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v67.l, v66.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v31.l, v15.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v66.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v15.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v30.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v28
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v69, v69
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v70, v70
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v31.h, v32.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v31.l, v15.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v32
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v29
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v30
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v26
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v6.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v31, v34
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v33, v51
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v29.l, v32.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v30.l, v15.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v32.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v15.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v26
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v27
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v24
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v5.l
; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s0
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v29.l, v66.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v102, v102
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v30.l, v15.l, s1
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s43, s17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v23.l, v7.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v5.h, v21.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v25, v25
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v22.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v48.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v23.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v21.h, v23.l, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v26, v26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21
-; GFX11-TRUE16-NEXT: s_and_b32 s4, vcc_lo, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v24.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v27, v25
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v21.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v26, v26
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v28
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v29
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xffff0000, v20
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v25, v25
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, s2
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v27, v26
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v21.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v4.h, v20.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v28, v28
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v22.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v23.l, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v20.h, v25.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v24.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v23.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v27, v29
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v25.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v28
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v30
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v28
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v21.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v65.l, v48.l, s4
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s3, v29, v27
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v28
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v25.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v22.l, v6.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v24.l, v23.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v20
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v28, v28
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v26.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v27
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v5.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v20.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v24, v24
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v26.l, v32.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v27.l, v15.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s57, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v20
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v25, v25
; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v19
-; GFX11-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v4.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v3.h, v19.h, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v20.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v113.l, v50.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v3.h, v19.h, s1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v25.l
; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v21.l, v5.l, s3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v4.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v22
; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v19.h, v22.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v3
-; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v22.l
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v19
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v24, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v28
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v29
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v19.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v4.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v26.l, v25.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v24, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v20.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v22.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v3.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v2.h, v18.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v24
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v21.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v18.h, v23.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v28
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v27, v26
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v23.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v25
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v22.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v27
-; GFX11-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v19.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v26, v25
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v50.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v3
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s56, s19
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v68.l, v49.l, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v20
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v19
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v4
+; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v27, v26
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v19.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v28, v25
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v22.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v24.l, v50.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v18
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v4.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v19
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v2.h, v18.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v20
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v24
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v26, v25
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v22.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v18.h, v27.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v4.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v27
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v2
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v21.l, v22.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v27, v27
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v19
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v25, v23
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v26, v26
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v28
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v24.l, v23.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v21
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v24.l, v27.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v18.l, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v25
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v21.l
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v24, v24
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v17
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v21
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v27.l
; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v22
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v1.h, v17.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v24, v24
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v18.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v25
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v23.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v17.h, v22.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v27
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v22.l
-; GFX11-TRUE16-NEXT: s_and_b32 s8, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v24.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s3
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v26, v25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v21.l, v23.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v28
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v1.h, v17.h, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v17.h, v24.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v25
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v21.l, v27.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v26, v26
; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v17
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v23, v21
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v17.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v26, v26
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v16
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v21, v21
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v16
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v0.h, v16.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v16.l, s2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v25, v25
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v1.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v16.h, v21.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v21.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s1, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v17.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v17
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v21, v21
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v16
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v16
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v23, v23
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v0.h, v16.h, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v21, v21
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v16.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v1.l, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v16.h, v23.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v23
; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v0.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v22.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v23.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v16.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v18.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v27, v25
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v24.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v16
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v24.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v27, v26
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v29, v28
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v31, v30
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v21
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v31, v30
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v25
; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v21.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v22.l, v23.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v24.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v0.l, s2
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v26
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v17.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v25
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v16.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v22.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX11-TRUE16-NEXT: s_and_b32 s9, s0, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v26
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v1.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v25
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v21.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v16
+; GFX11-TRUE16-NEXT: s_and_b32 s9, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v26
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v23.l
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v27
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v0.l
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v24.l, v22.l, s9
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v25.l, v24.l, s9
; GFX11-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v23.l, v21.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v21.l, v23.l, s2
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v16.l, v0.l, s3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v17.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v18.l, v2.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v18.l, v2.l, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v19.l, v3.l, s8
; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v20.l, v4.l, s7
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -11426,652 +11038,542 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: scratch_load_b32 v31, off, s32
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v36, 16, v14
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v13
; GFX12-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v30
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v15
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v36, v36
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v29
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v38, v38
; GFX12-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v15
; GFX12-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v30
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v34, v34
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v37, v37
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v30.l, s4
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v12
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v33, v33
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v39, v39
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v13.h, v29.h, s6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v29
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v10
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v26
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v6
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v32, v32
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v35, v35
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v14.h, v30.h, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v13
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v22
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v50, v50
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v50.l, v29.h, v33.l, s7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v36, 16, v14
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v34, v34
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v29
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v12
; GFX12-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v28
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v99, 0xffff0000, v22
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v49, v49
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v66, v66
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v67, v67
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s46, v100, v100
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.l, v30.h, v32.l, s3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v67.l, v14.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v100.l, v30.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v28
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v48, v48
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v52, v52
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v68, v68
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v101, v101
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v68.l, v33.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v101.l, v50.l
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v51, v51
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s45, v99, v99
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v12.h, v28.h, s10
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.l, v10.h, v26.h, s18
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v66.l, v32.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v99.l, v49.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v67, 16, v67
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v100
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v11
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v53, v53
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v29.l, s8
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v28.l, s12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v68
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v101
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v9
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.l, v28.h, v34.l, s11
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v53.l, v26.h, v36.l, s19
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v66, 16, v66
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v99
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s19, v67, v100
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v27
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v26
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v25
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v64, v64
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v26.l, s20
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s9
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s13
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s20, v68, v101
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v70, v70
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v70.l, v34.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v112.l, v51.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s18, v66, v99
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s19
; GFX12-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v11
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v65, v65
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v69, v69
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v71, v71
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v27.l, s16
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v69.l, v13.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v71.l, v12.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v103.l, v29.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v113.l, v28.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v10
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v35, v35
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v48, v48
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v50.l, v50.l, v33.l, s20
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v70
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v112
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.l, v49.l, v32.l, s18
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v67.l, v30.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v14.h, v30.h, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v30
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v36, v36
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v12
; GFX12-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v27
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v26
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v114, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v49, v49
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v50, v50
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v51, v51
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v54, v54
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v68, v68
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v29.l, s7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.l, v30.h, v32.l, s2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v11
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v37, v37
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v30.l, s3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v29
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v27
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v26
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v25
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.l, v9.h, v25.h, s22
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v69
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v71, 16, v71
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v103
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v113
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v68.l, v50.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s22, v70, v112
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v66.l, v49.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v67, 16, v67
; GFX12-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v8
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v52, v52
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v55, v55
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v66, v66
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v69, v69
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s59, v114, v114
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v12.h, v28.h, s9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v11.h, v27.h, s13
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v26.l, s19
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v32
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v51
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v28
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v9
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v115, 0xffff0000, v20
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v64, v64
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v15
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v102, 0xffff0000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v38, v38
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v25
; GFX12-TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v24
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v54, v54
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v80, v80
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v81, v81
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v81.l, v11.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s21
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.l, v25.h, v37.l, s23
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v115.l, v27.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s21, v69, v103
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s23, v71, v113
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v68
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v7
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v65, v65
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v67, v67
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v80, v80
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v82, v82
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.l, v51.l, v34.l, s22
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v66, 16, v66
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s19, 0, v67
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v82, v82
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v83, v83
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v25.l, s24
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v83.l, v10.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v81
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v117.l, v26.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v115
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s21
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s23
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s20, 0, v68
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.l, v10.h, v26.h, s17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v13
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v65.l, v28.h, v34.l, s10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v68.l, v27.h, v35.l, s14
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s20
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v29
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s20, v52, v114
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v53, v53
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v70, v70
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s60, v115, v115
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v27.l, s15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v30
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v103, 0xffff0000, v21
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v33, v33
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v39, v39
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s47, v102, v102
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v13.h, v29.h, s5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v24
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v23
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v84, v84
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v96, v96
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v70.l, v51.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s18, 0, v66
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v71, v71
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v81, v81
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v83, v83
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v84, v84
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v25.l, s23
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.l, v8.h, v24.h, s25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v66, 16, v34
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v35
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v71.l, v26.h, v36.l, s18
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v65
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v68
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s23, v64, v117
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.l, v8.h, v24.h, s26
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s25
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v83
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v117
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s25, v81, v115
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v69.l, v29.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v71.l, v28.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.l, v51.l, v32.l, s20
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.l, v9.h, v25.h, s21
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s16
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s21, v53, v115
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s56, v103, v103
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.l, v5.h, v21.h, s47
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.l, v29.h, v33.l, s6
; GFX12-TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v7
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v85, v85
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s43, v97, v97
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v24.l, s28
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v23.l, s42
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v70
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v98, 0xffff0000, v6
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v55, v55
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v55.l, v24.h, v38.l, s27
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s27, v83, v117
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v6
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v85, v85
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v24.l, s27
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v36
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s24
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v85.l, v24.h, v38.l, s26
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v71
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s24, v66, v118
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s26, v69, v128
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v51
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v129, 16, v27
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s25
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v69
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v71, 16, v71
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v33
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v102.l, v21.h, v49.l, s56
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v54
; GFX12-TRUE16-NEXT: v_and_b32_e32 v87, 0xffff0000, v23
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v85.l, v9.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s29
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v7.l, s43
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v119.l, v25.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s22, 0, v70
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s44, v98, v98
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s27
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v81.l, v27.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s21, 0, v69
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s23, 0, v71
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v86, v86
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v87, v87
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v87.l, v8.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v97.l, v7.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v85
-; GFX12-TRUE16-NEXT: v_mov_b16_e64 v129.l, v24.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e64 v131.l, v23.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v119, 16, v119
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v23
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v98, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v22
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v86, v86
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v96, v96
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s45, v100, v100
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v32.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s28
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v26
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s28, v80, v130
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v53.l, v65.l, v34.l, s24
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v29
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s20, 0, v69
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v5
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s27, v70, v129
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v30
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v82.l, v25.h, v37.l, s22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v49
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v102
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s22, v55, v116
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v87, v87
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v97, v97
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s43, v98, v98
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s46, v101, v101
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v48.l, v6.h, v22.h, s44
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v13.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v83.l, v26.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v81
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.l, v7.h, v23.h, s40
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v87
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v97
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v129, 16, v129
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v131
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s29, v85, v119
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v14.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v65.l, v22.h, v48.l, s45
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0, v11.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v83
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s25, 0, v81
-; GFX12-TRUE16-NEXT: s_and_b32 s5, s21, s5
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v64.l, v23.h, v39.l, s41
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s41, v87, v129
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s43, v97, v131
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s29
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.l, v7.h, v23.h, s29
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v23.l, s41
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v22.l, s45
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v13.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v135, 16, v24
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s29, v81, v131
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v53
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s23, 0, v80
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s20, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v21
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s57, v112, v112
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s21, 0, v70
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v29.l, v13.l, s5
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v12.l
-; GFX12-TRUE16-NEXT: s_and_b32 s3, s19, s3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v98.l, v48.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e64 v132.l, v65.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0, v10.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s27, 0, v83
-; GFX12-TRUE16-NEXT: s_and_b32 s9, s25, s9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v51.l, v32.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s56, v103, v52
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.l, v54.l, v33.l, s22
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v99, 0xffff0000, v22
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v48.l, v6.h, v22.h, s43
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v14.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v34.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v38
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v7.l, s42
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, s46
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v134, 16, v85
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s43, v87, v135
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s29
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s24, 0, v81
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s23, s4
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s58, v113, v113
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v28.l, s11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v21.l, s57
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v30.l, v14.l, s3
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s41
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v7.l, s43
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v85.l, v25.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v27.l, v11.l, s9
-; GFX12-TRUE16-NEXT: s_and_b32 s7, s23, s7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v98
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v132
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v11.h, v27.h, s14
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v32.l
-; GFX12-TRUE16-NEXT: s_and_b32 s11, s27, s11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v29.l, v13.l, s4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v96.l, v23.h, v39.l, s40
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v133, 16, v25
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v55.l, v71.l, v36.l, s28
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v71, 16, v52
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s44, v99, v99
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v145, 16, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v147, 16, v22
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s42, v86, v134
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s43
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v26
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s21, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s5, s24, s5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v67, 16, v12
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s58
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v28.l, v12.l, s7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v87.l, v24.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v97.l, v23.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v85
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v26.l, v10.l, s11
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s44, v98, v132
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.l, v27.h, v35.l, s15
-; GFX12-TRUE16-NEXT: s_and_b32 s2, s18, s2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0, v9.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v87
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v97
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s29, 0, v85
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v30.l, v14.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v53.l, v34.l, s5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v33.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v39
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v144, 16, v96
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s41, v84, v133
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s27
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s22, 0, v71
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0, v10.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v99.l, v22.h, v48.l, s44
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s45, v98, v145
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s47, v101, v147
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v24
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s29, 0, v86
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v21
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s44, v97, v144
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s41
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v27
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s22, s3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0, v8.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v48
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v146, 16, v99
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v7.l, s45
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, s47
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s43, 0, v98
+; GFX12-TRUE16-NEXT: s_and_b32 s10, s29, s10
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v49.l, v32.l, s2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v102, 0xffff0000, v5
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v22.l, s46
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v65.l, v65.l, v48.l, s44
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0, v8.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0, v7.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s41, 0, v87
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s43, 0, v97
-; GFX12-TRUE16-NEXT: s_and_b32 s13, s29, s13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v98.l, v65.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v52.l, v33.l, s3
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s57, v112, v114
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v26.l, v10.l, s10
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v11.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v66.l, v96.l, v39.l, s44
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v25
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s27, 0, v84
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s46, v100, v146
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.l, v68.l, v35.l, s26
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v68.l, v102.l, v49.l, s56
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v102, 16, v22
+; GFX12-TRUE16-NEXT: s_and_b32 s14, s43, s14
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v50.l, v4.h, v20.h, s59
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s57
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v25.l, v9.l, s13
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v21
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s15, s41, s15
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v33.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v24.l, v8.l, s14
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s12
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0, v9.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s41, 0, v96
+; GFX12-TRUE16-NEXT: s_and_b32 s8, s27, s8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0, v7.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 0, v6.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s45, 0, v100
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s47, 0, v102
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v24.l, v8.l, s15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v98
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v80.l, v35.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v114.l, v52.l
-; GFX12-TRUE16-NEXT: s_and_b32 s4, s20, s4
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v35.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v27.l, v11.l, s8
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v113.l, v20.h, v50.l, s60
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v21
+; GFX12-TRUE16-NEXT: s_and_b32 s12, s41, s12
+; GFX12-TRUE16-NEXT: s_and_b32 s16, s45, s16
+; GFX12-TRUE16-NEXT: s_and_b32 s18, s47, s18
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v50.l, v33.l, s4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v80
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v114
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v82.l, v36.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v84.l, v37.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v86.l, v38.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v96.l, v39.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s24, v80, v114
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v116.l, v53.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v118.l, v54.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e64 v128.l, v55.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e64 v130.l, v64.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.l, v52.l, v35.l, s24
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v82
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v84
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v80.l, v52.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v116
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v118
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v128
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v130
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v80
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s26, v82, v116
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s28, v84, v118
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s40, v86, v128
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s42, v96, v130
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s24, 0, v80
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v25.l, v9.l, s12
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s57, 0, v112
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v23.l, v7.l, s16
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v22.l, v6.l, s18
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v113
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v50
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v119, 16, v28
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v37
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v82
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v68
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v12.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s25, v67, v119
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 0, v49.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s40, v83, v132
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s56, 0, v103
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v54
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v53.l, v53.l, v36.l, s26
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.l, v54.l, v37.l, s28
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v55.l, v55.l, v38.l, s40
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v64.l, v64.l, v39.l, s42
-; GFX12-TRUE16-NEXT: s_and_b32 s8, s24, s8
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v82.l, v53.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s25
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v35.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v64.l, v82.l, v37.l, s40
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v65.l, v85.l, v38.l, s42
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s26, 0, v83
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v28
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v55
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v67.l, v99.l, v48.l, s46
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0, v36.l
+; GFX12-TRUE16-NEXT: s_and_b32 s7, s26, s7
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s25, 0, v82
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v52.l, v35.l, s8
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v84.l, v54.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v86.l, v55.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v96.l, v64.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v34.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v82
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v84
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96
-; GFX12-TRUE16-NEXT: s_and_b32 s6, s22, s6
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0, v36.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0, v37.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0, v38.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0, v39.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s26, 0, v82
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s28, 0, v84
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s40, 0, v86
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s42, 0, v96
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v54.l, v35.l, s7
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s28, 0, v85
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v64
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v65
+; GFX12-TRUE16-NEXT: s_and_b32 s6, s25, s6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v66
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v51.l, v34.l, s6
-; GFX12-TRUE16-NEXT: s_and_b32 s10, s26, s10
-; GFX12-TRUE16-NEXT: s_and_b32 s12, s28, s12
-; GFX12-TRUE16-NEXT: s_and_b32 s14, s40, s14
-; GFX12-TRUE16-NEXT: s_and_b32 s16, s42, s16
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v28.l, v12.l, s6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v67
+; GFX12-TRUE16-NEXT: s_and_b32 s9, s28, s9
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0, v37.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0, v38.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0, v39.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0, v48.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s40, 0, v87
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s42, 0, v97
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s44, 0, v99
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s46, 0, v101
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v53.l, v36.l, s10
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v54.l, v37.l, s12
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v55.l, v38.l, s14
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v64.l, v39.l, s16
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v55.l, v36.l, s9
+; GFX12-TRUE16-NEXT: s_and_b32 s11, s40, s11
+; GFX12-TRUE16-NEXT: s_and_b32 s13, s42, s13
+; GFX12-TRUE16-NEXT: s_and_b32 s15, s44, s15
+; GFX12-TRUE16-NEXT: s_and_b32 s17, s46, s17
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v64.l, v37.l, s11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v65.l, v38.l, s13
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v66.l, v39.l, s15
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v67.l, v48.l, s17
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v31
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v31
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v66.l, v15.h, v31.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v67, v67
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v68, v68
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v68.l, v66.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v69.l, v15.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v31
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v31
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v15.h, v31.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v69, v69
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v70, v70
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v15
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v31.h, v32.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v67.l, v31.h, v66.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.l, v31.l, v15.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v68
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v69
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v70.l, v67.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v71.l, v31.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v70
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v71, 16, v71
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v31.l, v15.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v32
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v29
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v30
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v68, v70
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v69, v71
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v31, v34
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v33, v51
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v29.l, v32.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v30.l, v15.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v32.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v15.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v26
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v27
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v67.l, v66.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v31.l, v15.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v66.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v15.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v29.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v30.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v27
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v28
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v26
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v27
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v6.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v24
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v5.l
; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s0
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s3, s1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v29.l, v66.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v102, v102
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v30.l, v15.l, s1
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s43, s17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v23.l, v7.l, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v5.h, v21.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v25, v25
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v22.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v48.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v29.l, v23.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v21.h, v23.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v26, v26
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21
-; GFX12-TRUE16-NEXT: s_and_b32 s4, vcc_lo, s1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v24.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v27, v25
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v21.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v26, v26
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v28
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v29
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v28, 0xffff0000, v20
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v25, v25
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, s2
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v27, v26
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v5.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v21.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v4.h, v20.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v28, v28
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v22.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v23.l, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v26
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v20.h, v25.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v30.l, v24.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v23.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v27, v29
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v26.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v29.l, v25.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v28
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v30
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v26.l, v32.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v27.l, v15.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s57, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v20
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v25, v25
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v19
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v20.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v28
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v21.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v65.l, v48.l, s4
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s3, v29, v27
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v28
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v4
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v25.l, s3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v22.l, v6.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v24.l, v23.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v20
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v28, v28
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v26.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v27
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v5.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v113.l, v50.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v20.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v24, v24
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v19
-; GFX12-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v4.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v3.h, v19.h, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v3.h, v19.h, s1
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v25.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v21.l, v5.l, s3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v20.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v4.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v19.h, v22.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v3
-; GFX12-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v21.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v29.l, v22.l
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v19
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v24, v23
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v28
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v29
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v19.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v22
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v4.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v19.h, v22.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v50.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v3
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s56, s19
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v26.l, v25.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v24, v23
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v68.l, v49.l, s2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v20
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v19
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v4
+; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v27, v26
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v19.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v28, v25
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v18
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v20.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v22.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v24.l, v50.l, s2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v22.l, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v19.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v3.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v18
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v4.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v19
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v2.h, v18.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v24
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v21.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v2.h, v18.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v20
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v24
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v26, v25
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v22.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v18.h, v23.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v25
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v28
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v27, v26
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v24.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v23.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v25
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v22.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v18.h, v27.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v4.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v26
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v27
-; GFX12-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v19.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v26, v25
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, s2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v27
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v2
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX12-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v21.l, v22.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v27, v27
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v19
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v25, v23
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v26, v26
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v28
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v21
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v24.l, v23.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v24.l, v27.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v18.l, s0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v25
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v21.l
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v24, v24
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v17
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v21
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v27.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v22
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v1.h, v17.h, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v24, v24
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v18.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v25
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v23.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v17.h, v22.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v26
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v27
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v22.l
-; GFX12-TRUE16-NEXT: s_and_b32 s8, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v24.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s3
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v26, v25
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v1.h, v17.h, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v18
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v17.h, v24.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v25
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v21.l, v23.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v27
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v28
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v21.l, v27.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v26, v26
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v17
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v23, v21
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v17.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v26, v26
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v16
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v21, v21
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v16
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s8, s1, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v17.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v17
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v21, v21
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v16
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v16
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v23, v23
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v0.h, v16.h, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v16.l, s2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v25, v25
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v17.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v1.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v16.h, v21.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v29.l, v21.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v0.h, v16.h, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v21, v21
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v16.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v1.l, s2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v16.h, v23.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v23
; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v31.l, v0.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v22.l, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v23.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v30.l, v16.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v18.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v27, v25
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v24.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v18
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v16
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v24.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v27, v26
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v29, v28
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v31, v30
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v21
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v31, v30
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v25
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v21.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v22.l, v23.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v24.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v0.l, s2
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v26
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v17.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v25
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v23.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v16.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v22.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX12-TRUE16-NEXT: s_and_b32 s9, s0, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v26
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v1.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v25
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v21.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v16
+; GFX12-TRUE16-NEXT: s_and_b32 s9, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v26
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v23.l
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v27
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v0.l
-; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v24.l, v22.l, s9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v25.l, v24.l, s9
; GFX12-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v23.l, v21.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v21.l, v23.l, s2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v16.l, v0.l, s3
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v17.l, v1.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v18.l, v2.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v18.l, v2.l, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v19.l, v3.l, s8
; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v20.l, v4.l, s7
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -12832,27 +12334,24 @@ define bfloat @v_maximumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
; GFX11-TRUE16-LABEL: v_maximumnum_bf16_no_ieee:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
@@ -12890,30 +12389,28 @@ define bfloat @v_maximumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -13141,17 +12638,11 @@ define <2 x bfloat> @v_maximumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v7
@@ -13160,11 +12651,8 @@ define <2 x bfloat> @v_maximumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
@@ -13241,17 +12729,11 @@ define <2 x bfloat> @v_maximumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v4, v6
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v7
@@ -13262,11 +12744,8 @@ define <2 x bfloat> @v_maximumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v2.l
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
@@ -13580,61 +13059,51 @@ define <3 x bfloat> @v_maximumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v8, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v7, v7
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v4.l, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v4.l, s3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v9
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v10, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v9
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v8, v11
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v6.l, v4.l, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, s0
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
; GFX11-TRUE16-NEXT: s_and_b32 s0, s3, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
; GFX11-TRUE16-NEXT: s_and_b32 s1, s4, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s0
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v3bf16_no_ieee:
@@ -13709,66 +13178,55 @@ define <3 x bfloat> @v_maximumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v8, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v7, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, s0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s3
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v4.l, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v4.l, s3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v3.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v8
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v9
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v10, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v9
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v10
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v8, v11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v6.l, v4.l, s0
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
; GFX12-TRUE16-NEXT: s_and_b32 s0, s3, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
; GFX12-TRUE16-NEXT: s_and_b32 s1, s4, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s0
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v3bf16_no_ieee:
@@ -14166,75 +13624,62 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v1.h, v3.h, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v10, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v4, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v4.l, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.h, v5.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v6.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.h, v6.l, s3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v10, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v9, v12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v7
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v10
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v6.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v8, v13
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v13, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v6.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v5.l
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v8
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
; GFX11-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v4.l, s5
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v5.l, s5
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v6.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v6.l, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v4bf16_no_ieee:
@@ -14326,84 +13771,71 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v1.h, v3.h, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v10, v10
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v4, v4
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v4.l, s0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.h, v5.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.h, v6.l, s3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v6.l, s3
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v6.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v10, v8
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v9, v12
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v7
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v10
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v6.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v8, v13
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v13, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v9
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v6.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v5.l
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v8
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
; GFX12-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v4.l, s5
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v5.l, s5
; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v6.l, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v6.l, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v4bf16_no_ieee:
diff --git a/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
index 73e9112118509..c8531a7913ab4 100644
--- a/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
@@ -109,27 +109,24 @@ define bfloat @v_minimumnum_bf16(bfloat %x, bfloat %y) #1 {
; GFX11-TRUE16-LABEL: v_minimumnum_bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
@@ -167,30 +164,28 @@ define bfloat @v_minimumnum_bf16(bfloat %x, bfloat %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -303,18 +298,19 @@ define bfloat @v_minimumnum_bf16_nnan(bfloat %x, bfloat %y) #1 {
; GFX11-TRUE16-LABEL: v_minimumnum_bf16_nnan:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -341,18 +337,20 @@ define bfloat @v_minimumnum_bf16_nnan(bfloat %x, bfloat %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -578,17 +576,11 @@ define <2 x bfloat> @v_minimumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) #1 {
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v7
@@ -597,11 +589,8 @@ define <2 x bfloat> @v_minimumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) #1 {
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
@@ -678,17 +667,11 @@ define <2 x bfloat> @v_minimumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) #1 {
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v6
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v7
@@ -699,11 +682,8 @@ define <2 x bfloat> @v_minimumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) #1 {
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
@@ -905,12 +885,9 @@ define <2 x bfloat> @v_minimumnum_v2bf16_nnan(<2 x bfloat> %x, <2 x bfloat> %y)
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v4
@@ -968,12 +945,9 @@ define <2 x bfloat> @v_minimumnum_v2bf16_nnan(<2 x bfloat> %x, <2 x bfloat> %y)
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v4
@@ -1279,61 +1253,51 @@ define <3 x bfloat> @v_minimumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) #1 {
; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v8, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v7, v7
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v4.l, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v4.l, s3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v9
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v10, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v9
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v8, v11
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v6.l, v4.l, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, s0
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
; GFX11-TRUE16-NEXT: s_and_b32 s0, s3, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
; GFX11-TRUE16-NEXT: s_and_b32 s1, s4, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s0
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v3bf16:
@@ -1408,66 +1372,55 @@ define <3 x bfloat> @v_minimumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) #1 {
; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v8, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v7, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, s0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s3
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v4.l, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v4.l, s3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v3.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v8
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v9
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v10, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v9
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v10
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v8, v11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v6.l, v4.l, s0
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
; GFX12-TRUE16-NEXT: s_and_b32 s0, s3, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
; GFX12-TRUE16-NEXT: s_and_b32 s1, s4, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s0
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v3bf16:
@@ -1728,19 +1681,14 @@ define <3 x bfloat> @v_minimumnum_v3bf16_nnan(<3 x bfloat> %x, <3 x bfloat> %y)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v2.h, v0.h, s0
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.h
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v7
; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
; GFX11-TRUE16-NEXT: s_and_b32 s0, s3, s0
@@ -1810,19 +1758,14 @@ define <3 x bfloat> @v_minimumnum_v3bf16_nnan(<3 x bfloat> %x, <3 x bfloat> %y)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v2.h, v0.h, s0
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.h
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v7
; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
; GFX12-TRUE16-NEXT: s_and_b32 s0, s3, s0
@@ -2215,75 +2158,62 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) #1 {
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v1.h, v3.h, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v10, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v4, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v4.l, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.h, v5.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v6.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.h, v6.l, s3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v10, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v9, v12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v7
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v10
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v6.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v8, v13
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v13, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v6.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v5.l
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v8
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
; GFX11-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v4.l, s5
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v5.l, s5
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v6.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v6.l, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v4bf16:
@@ -2375,84 +2305,71 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) #1 {
; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v1.h, v3.h, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v10, v10
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v4, v4
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v4.l, s0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.h, v5.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.h, v6.l, s3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v6.l, s3
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v6.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v10, v8
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v9, v12
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v7
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v10
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v6.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v8, v13
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v13, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v9
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v6.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v5.l
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v8
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
; GFX12-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v4.l, s5
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v5.l, s5
; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v6.l, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v6.l, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v4bf16:
@@ -2781,41 +2698,36 @@ define <4 x bfloat> @v_minimumnum_v4bf16_nnan(<4 x bfloat> %x, <4 x bfloat> %y)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v0
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v6
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v9, v8
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v0.h
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v11, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.h, v1.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v3.h, v1.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v1.h
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v4
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v0.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v7
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v9
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v1.h, s0
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, s4
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s5, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v0.h, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v2.h, v0.h, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.h
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s0, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s4, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v6.l, v1.h, s5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v4bf16_nnan:
@@ -2884,45 +2796,41 @@ define <4 x bfloat> @v_minimumnum_v4bf16_nnan(<4 x bfloat> %x, <4 x bfloat> %y)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v0
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v6
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v9, v8
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v0.h
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v11, v10
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.h, v1.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v3.h, v1.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v1.h
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v0.h, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.h
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v1.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v7
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v8
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v9
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v2.h, v0.h, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.h
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
+; GFX12-TRUE16-NEXT: s_and_b32 s5, s0, s2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s3, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s4, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v1.h, s0
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s2
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s3, s4
-; GFX12-TRUE16-NEXT: s_and_b32 s2, s5, s6
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v6.l, v1.h, s5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, s1
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v0.h, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v4bf16_nnan:
@@ -3475,110 +3383,92 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) #1 {
; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v5.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v5.h, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v11, v11
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v12, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v5.h, v6.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v5.h, v7.l, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v13, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v10, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v16
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v11, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v1.h, v4.h, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v1.h, v4.h, s1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s5, v8, v9
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v14, v14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v4.h, v8.l, s2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v15, v15
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v16, v16
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v12, v12
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v15, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v16.l, v7.l, s5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v4.h, v6.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v7.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v14, v14
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s5, v9, v12
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v10.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v17, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v0.h, v3.h, s5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v15
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v0.h, v3.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s2, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v14, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v7.l, v6.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v4.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v3.h, v9.l, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v10.l, v8.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v9.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v8.l, v7.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v3.h, v9.l, s6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v13, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v12, v11
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v14, v13
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v16, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v12
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v16, v15
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v10
; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v9.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v8.l, v9.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.l
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v4.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v8.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v2.l
+; GFX11-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v11
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v9.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT: s_and_b32 s7, s0, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v11
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v1.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v12
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v7.l, v8.l, s7
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v10.l, v6.l, s7
; GFX11-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v9.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v9.l, s2
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, s3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v2.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v2.l, s1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v6bf16:
@@ -3713,122 +3603,104 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) #1 {
; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v4
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v5.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v5.h, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v11, v11
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v12, v12
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v5.h, v6.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v5.h, v7.l, s0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v13, v13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v10, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v16
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v11, v11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v1.h, v4.h, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v9
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v1.h, v4.h, s1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s5, v8, v9
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v14, v14
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v4.h, v8.l, s2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v15, v15
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v16, v16
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v8.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v10.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v17, v17
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v12, v12
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v15, v15
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v0.h, v3.h, s5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v15
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v16.l, v7.l, s5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v4.h, v6.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v7.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v8
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v14, v14
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s5, v9, v12
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v0.h, v3.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s2, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s4
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v2.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v14, v11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v7.l, v6.l, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v4.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v3.h, v9.l, s6
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s8
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v8.l, v7.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v3.h, v9.l, s6
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v13, v14
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v10.l, v8.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v9.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v11
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v1.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v6.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v3.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v0.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v10
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v12, v11
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v14, v13
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v16, v15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v12
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v16, v15
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v10
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v9.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v8.l, v9.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.l
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v11
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v4.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v6.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v8.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v2.l
+; GFX12-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v11
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v9.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX12-TRUE16-NEXT: s_and_b32 s7, s0, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v11
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v1.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v12
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v7.l, v8.l, s7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v10.l, v6.l, s7
; GFX12-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.l, v9.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v9.l, s2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, s3
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v2.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v2.l, s1
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v6bf16:
@@ -4611,149 +4483,125 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) #1 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v3.h, v7.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v7.h, v8.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v3.h, v7.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v7.h, v9.l, s0
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v7.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v9
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v3.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v2.h, v6.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v7.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v6.h, v10.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v15, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v11.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v8.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v14, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v16
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v18
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v9.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v2.h, v6.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v9.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v6.h, v8.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v9.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v15, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v16
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v14, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v10
; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v3.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v14, v16
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v15, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v7.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v10.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v11.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v10.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v12, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v9.l, v8.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v1.h, v5.h, s2
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v11, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v6.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v16.l, v8.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, s2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v10.l, v9.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v1.h, v5.h, s4
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v3.l
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v5.h, v8.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v9.l
-; GFX11-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v5.h, v10.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v8.l
+; GFX11-TRUE16-NEXT: s_and_b32 s7, s0, s1
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v10
; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s3
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v13, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v11.l, v10.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v7.l, v3.l, s7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v12.l, v8.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v14, v14
; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v5
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v10, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v0.h, v4.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v12, v12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v1.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v4.h, v10.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v10.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v11, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v0.h, v4.h, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v13.l, v10.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v4.h, v11.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v0.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v8.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v11.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v4.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v6.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v14, v12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v6
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v14, v13
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v16, v15
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v18, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v12
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v18, v17
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v10.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v11.l, s0
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v5.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v4.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v8.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v10.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v8
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11-TRUE16-NEXT: s_and_b32 s8, s0, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v13
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v1.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v12
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v10.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4
+; GFX11-TRUE16-NEXT: s_and_b32 s8, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v13
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v11.l
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v14
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v9.l, v8.l, s8
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v12.l, v10.l, s8
; GFX11-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v11.l, v10.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v11.l, s2
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.l, v0.l, s3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v6.l, v2.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v7.l, v3.l, s7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v6.l, v2.l, s1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v8bf16:
@@ -4913,171 +4761,145 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v11, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v6
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v3.h, v7.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v3.h, v7.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v7.h, v8.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v7.h, v9.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v7.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v9
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v3.l, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v2.h, v6.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v11
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v7.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v2.h, v6.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v9.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v6.h, v10.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v15, v12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v16
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v10.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v11.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v6.h, v8.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v8.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v14, v12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v16
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v18
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v9.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v9.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v15, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v16
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v14, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v10
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v3.l, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v14, v16
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v15, v15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v18
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v7.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v11, v15
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v10.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v14
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v11.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v10.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v12, v12
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v6.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v16.l, v8.l, s1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s3, s2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v9.l, v8.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v14
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v1.h, v5.h, s2
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v6.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v10.l, v9.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v12
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v1.h, v5.h, s4
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v3.l
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v9
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v5.h, v8.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v8.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v9.l
-; GFX12-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v5.h, v10.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v8.l
+; GFX12-TRUE16-NEXT: s_and_b32 s7, s0, s1
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v10
; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s3
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v13, v12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v11.l, v10.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v15
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v7.l, v3.l, s7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v12.l, v8.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v14, v14
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v5
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v10, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v0.h, v4.h, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v12, v12
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v5.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v1.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v11, v11
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v0.h, v4.h, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, s2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v4.h, v10.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v10.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v13.l, v10.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v4.h, v11.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v0.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v8.l, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v11.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v4.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v6.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v14, v12
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v9.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v6
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v14, v13
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v16, v15
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v18, v17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v12
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v18, v17
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v10.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v11.l, s0
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v5.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v12
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v11.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v4.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v8.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v10.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v8
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX12-TRUE16-NEXT: s_and_b32 s8, s0, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v13
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v1.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v12
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v10.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4
+; GFX12-TRUE16-NEXT: s_and_b32 s8, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v13
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v11.l
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v14
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v9.l, v8.l, s8
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v12.l, v10.l, s8
; GFX12-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v11.l, v10.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v11.l, s2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.l, v0.l, s3
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v1.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v6.l, v2.l, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v7.l, v3.l, s7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v6.l, v2.l, s1
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v8bf16:
@@ -6496,303 +6318,250 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) #
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v7
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v15
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v24, v24
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v7.h, v15.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v7.h, v15.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v15
; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v15.l, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v14.l, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v16.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v15.h, v16.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v16.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v17.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v7.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v15.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v6.h, v14.h, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v20, v21
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v14.h, v18.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v16.l, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v21, v20
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v18.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v18
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v15.h, v18.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v18.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v14.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v17
; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v7.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v14
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v7.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v20, v19
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v6.h, v14.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v17.l, v18.l, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v21
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v14.h, v21.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v20, v19
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v15.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v21, v20
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v17
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v7.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v19
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v22
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v6.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v18.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v17.l, v16.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v14
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v20
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v19.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v16, v16
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v17
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v15.l, v7.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v5.h, v13.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v14.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v16
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v15.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v13.h, v15.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v13
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s0, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v16.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v21, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v13.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v23
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v22, v20
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v15
+; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v21.l, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v20, v20
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v16.l, v18.l, s2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v19
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v14
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s3, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v18, v18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v20, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v5.h, v13.h, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v13.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v18, v18
; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s2
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v21, v20
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v13.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v4.h, v12.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v14.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v20
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v15.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v12.h, v17.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v15.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v21, v23
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v16.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v22
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v24
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v21.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v13.h, v19.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v15.l, v7.l, s4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v13
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v24, v24
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v20, v18
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v17.l, v21.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v15
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v23, v22
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v19.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v19.l, v18.l, s4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s3, v23, v21
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v22
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v13.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v4.h, v12.h, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v15, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v16
; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v20.l, v17.l, s3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v22
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13
; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v14.l, v6.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v16.l, v15.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v18.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v12.h, v17.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v15
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v19.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s1, s2
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v18, v15
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v20, v20
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v12
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v19
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v5.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v17.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v12.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v15, v15
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v16, v16
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v3.h, v11.h, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v16.l, v19.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v13.l, v5.l, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v11.h, v15.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v13
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v16, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v4
+; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v20, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v11.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v21, v18
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v14.l, v17.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v15.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v2.h, v10.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v12
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v16
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v18, v17
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v15.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v10.h, v19.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v3.h, v11.h, s2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v15, v15
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v17.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v13.l, v5.l, s3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v4.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v11.h, v14.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v3
-; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v14.l
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v11
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v16, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v20
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v21
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v11.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v18.l, v17.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v16, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v12.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v14.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v3.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v2.h, v10.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v13.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v10.h, v15.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v20
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v19, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v15.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v17
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v14.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v11.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v18, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v13.l, v14.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v19, v19
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v4.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v19
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v2
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v13.l, v15.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v11
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v17, v14
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v20
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v16.l, v15.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v16.l, v19.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v10.l, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v17
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v13.l
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v16, v16
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v15, v15
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v13
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v19.l
; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v1.h, v9.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v16, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v10.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v17
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v15.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v9.h, v14.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v14.l
-; GFX11-TRUE16-NEXT: s_and_b32 s8, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v16.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s3
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v18, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v13.l, v15.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v20
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v1.h, v9.h, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v9.h, v16.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v15, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v13.l, v19.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v15, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v18, v18
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v13, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v0.h, v8.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v8.l, s2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v17, v17
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v1.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v8.h, v13.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v13.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v15, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s1, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v9
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v15, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v0.h, v8.h, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v8.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v8.h, v15.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v15
; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v0.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v14.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v15.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v8.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v10.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v19, v17
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v16.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v16.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v19, v18
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v21, v20
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v23, v22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v23, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v17
; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v13.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v14.l, v15.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v16.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s2
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v9.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v17
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v8.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v14.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; GFX11-TRUE16-NEXT: s_and_b32 s9, s0, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v18
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v1.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v17
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v13.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v8
+; GFX11-TRUE16-NEXT: s_and_b32 s9, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v14
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v18
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v15.l
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v19
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v16.l, v14.l, s9
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v17.l, v16.l, s9
; GFX11-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v15.l, v13.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v13.l, v15.l, s2
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, s3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v9.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v10.l, v2.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v10.l, v2.l, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v11.l, v3.l, s8
; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v12.l, v4.l, s7
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -7119,346 +6888,291 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) #
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v7
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v7
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v15
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v24, v24
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v7.h, v15.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v7.h, v15.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v15
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v15.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v14.l, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v16.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v18
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v15.h, v16.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v7.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v16.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v17.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v15.h, v18.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v18.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v14.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v17
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v7.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v18
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v15.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v6.h, v14.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v20, v21
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v22
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v23
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v14
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v7.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v20, v19
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v15
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v14.h, v18.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v6.h, v14.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v16.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v21, v20
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v19.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v18.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v17.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v17.l, v18.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v21
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v7.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v15.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v21, v20
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v22
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v6.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v23
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v18.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v17.l, v16.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v14
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v20
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v7.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v19.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v16, v16
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v17
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v13
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v14.h, v21.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v20, v19
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v16
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v17
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v7.l, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v19
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v22, v20
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v14
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v15
+; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v21.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v20, v20
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v19
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v15.l, v7.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v5
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v5.h, v13.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v14.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v16
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v18.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v15.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v13.h, v15.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v13
-; GFX12-TRUE16-NEXT: s_and_b32 s4, s0, s1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v16.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v21, v17
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v13.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v22
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v23
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v12
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v16.l, v18.l, s2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v17
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v6
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v19
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v14
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v13
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s3, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v18, v18
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v20, v19
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v5.h, v13.h, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v13.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v18, v18
; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s2
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v21, v20
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v5.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v13.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v4.h, v12.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v14.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v20
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v21.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v13.h, v19.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v19
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v15.l, s2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v12.h, v17.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v15.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v21, v23
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v17.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v20.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v16.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v22
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v15.l, v7.l, s4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v13
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v24, v24
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v20, v18
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v17.l, v21.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v15
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v23, v22
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v12
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v24
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v19.l, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v19.l, v18.l, s4
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s3, v23, v21
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v22
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v13.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v4.h, v12.h, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v15, v15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v16
; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v20.l, v17.l, s3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v22
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v5.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v14.l, v6.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v16.l, v15.l, s1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v18.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v12.h, v17.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v15
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v19.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v18
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s1, s2
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v18, v15
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v20, v20
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v12
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v19
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v5.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v11
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v17.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v12.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v15, v15
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v16, v16
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v11
-; GFX12-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v14
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v3.h, v11.h, s2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v15, v15
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v17.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v3.h, v11.h, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v14
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v13.l, v5.l, s3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v15.l, v12.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v4.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v16.l, v19.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v13.l, v5.l, s2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v11.h, v14.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v3
-; GFX12-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v13.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v14.l
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v11
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v16, v15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v20
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v21
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v11.h, v15.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v11.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v17.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v13
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v4
+; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v20, v19
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v11.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v21, v18
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v18.l, v17.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v16, v15
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v14.l, v17.l, s2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v10
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v16.l, v12.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v15.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v14.l, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v11.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v3.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v2.h, v10.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v16
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v13.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v2.h, v10.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v12
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v16
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v18, v17
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v15.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v10.h, v15.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v20
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v19, v18
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v16.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v15.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v17
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v14.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v10.h, v19.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v14
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v4.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v19
-; GFX12-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v11.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v18, v17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, s2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v19
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v2
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX12-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v13.l, v14.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v19, v19
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v13.l, v15.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v11
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v17, v14
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v20
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v16.l, v15.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v16.l, v19.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v10.l, s0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v17
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v14.l, v13.l
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v16, v16
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v9
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v15, v15
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v13
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v19.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v14
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v1.h, v9.h, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v16, v16
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v10.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v17
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v15.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v9.h, v14.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v19
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v14.l
-; GFX12-TRUE16-NEXT: s_and_b32 s8, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v16.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s3
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v18, v17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v1.h, v9.h, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v9.h, v16.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v15, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v17
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v13.l, v15.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v19
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v20
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v13.l, v19.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v9
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v15, v13
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v18, v18
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v13, v13
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v8
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v15, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s8, s1, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v9
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v8
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v8
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v15, v15
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v0.h, v8.h, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v8.l, s2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v17, v17
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v9.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v1.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v8.h, v13.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v21.l, v13.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v0.h, v8.h, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v8.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, s2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v8.h, v15.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v15
; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v0.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v14.l, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v20.l, v15.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v8.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v10.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v19, v17
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v16.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v8
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v16.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v19, v18
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v21, v20
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v23, v22
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v23, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v17
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v13.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v14.l, v15.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v16.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s2
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v18.l, v9.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v17
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v17.l, v15.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v19.l, v8.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v14.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; GFX12-TRUE16-NEXT: s_and_b32 s9, s0, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v18
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v1.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v17
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v13.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v8
+; GFX12-TRUE16-NEXT: s_and_b32 s9, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v14
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v18
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v15.l
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v19
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v16.l, v14.l, s9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v17.l, v16.l, s9
; GFX12-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v15.l, v13.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v13.l, v15.l, s2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, s3
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v9.l, v1.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v10.l, v2.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v10.l, v2.l, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v11.l, v3.l, s8
; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v12.l, v4.l, s7
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -10289,594 +10003,492 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v36, 16, v14
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v13
; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v30
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v15
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v36, v36
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v29
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v38, v38
; GFX11-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v15
; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v30
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v34, v34
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v37, v37
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v30.l, s4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v33, v33
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v39, v39
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v13.h, v29.h, s6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v36, 16, v14
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v34, v34
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v29
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v12
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v28
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v35, v35
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v48, v48
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v14.h, v30.h, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v30
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v36, v36
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v12
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v27
; GFX11-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v26
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v114, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v49, v49
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v50, v50
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v51, v51
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v54, v54
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v68, v68
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v29.l, s7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.l, v30.h, v32.l, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v11
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v37, v37
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v30.l, s3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v29
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v27
; GFX11-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v32, v32
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v35, v35
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v14.h, v30.h, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v22
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v50, v50
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v50.l, v29.h, v33.l, s7
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v28
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v99, 0xffff0000, v22
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v49, v49
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v66, v66
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v67, v67
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s46, v100, v100
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.l, v30.h, v32.l, s3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v100.l, v30.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v9
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v52, v52
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v55, v55
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v66, v66
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v69, v69
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s59, v114, v114
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v12.h, v28.h, s9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v11.h, v27.h, s13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v26.l, s19
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v32
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v51
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v28
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v48, v48
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v52, v52
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v68, v68
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v101, v101
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v101.l, v50.l
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v51, v51
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s45, v99, v99
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v12.h, v28.h, s10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.l, v10.h, v26.h, s18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v32.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v99.l, v49.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v67, 16, v67
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v100
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v11
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v53, v53
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v29.l, s8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v28.l, s12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v68
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v101
; GFX11-TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.l, v28.h, v34.l, s11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v53.l, v26.h, v36.l, s19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v66, 16, v66
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v99
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s19, v67, v100
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v26
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v115, 0xffff0000, v20
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v64, v64
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v15
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v102, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v38, v38
; GFX11-TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v25
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v64, v64
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v26.l, s20
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s13
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s20, v68, v101
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v70, v70
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v34.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v112.l, v51.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s18, v66, v99
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s19
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v65, v65
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v69, v69
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v71, v71
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v27.l, s16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v103.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v113.l, v28.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v50.l, v50.l, v33.l, s20
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v70
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v112
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.l, v49.l, v32.l, s18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v30.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v9
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v25
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.l, v9.h, v25.h, s22
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v69
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v71, 16, v71
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v103
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v113
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v50.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s22, v70, v112
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v49.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v67, 16, v67
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v8
; GFX11-TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v24
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v54, v54
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v80, v80
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v81, v81
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.l, v11.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s21
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.l, v25.h, v37.l, s23
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v115.l, v27.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s21, v69, v103
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s23, v71, v113
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v68
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.l, v51.l, v34.l, s22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v66, 16, v66
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s19, 0, v67
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v82, v82
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v83, v83
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v25.l, s24
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.l, v10.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v81
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v117.l, v26.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v115
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s21
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s23
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s20, 0, v68
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v65, v65
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v67, v67
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v80, v80
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v82, v82
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.l, v10.h, v26.h, s17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v65.l, v28.h, v34.l, s10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v68.l, v27.h, v35.l, s14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s20
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v29
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s20, v52, v114
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v53, v53
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v70, v70
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s60, v115, v115
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v27.l, s15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v30
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v103, 0xffff0000, v21
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v33, v33
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v39, v39
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s47, v102, v102
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v13.h, v29.h, s5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v24
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v23
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v84, v84
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v96, v96
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v51.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s18, 0, v66
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.l, v8.h, v24.h, s26
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v83
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v117
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s25, v81, v115
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v28.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v71, v71
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v81, v81
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v83, v83
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v84, v84
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v25.l, s23
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.l, v8.h, v24.h, s25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v66, 16, v34
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v35
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v71.l, v26.h, v36.l, s18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v65
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v68
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s23, v64, v117
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.l, v51.l, v32.l, s20
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.l, v9.h, v25.h, s21
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s16
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s21, v53, v115
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s56, v103, v103
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.l, v5.h, v21.h, s47
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.l, v29.h, v33.l, s6
; GFX11-TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v85, v85
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s43, v97, v97
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v24.l, s28
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v23.l, s42
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v70
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v98, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v55, v55
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v55.l, v24.h, v38.l, s27
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s27, v83, v117
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v69
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v71, 16, v71
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v85, v85
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v24.l, s27
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v36
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s24
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v85.l, v24.h, v38.l, s26
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v71
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s24, v66, v118
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s26, v69, v128
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v51
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v129, 16, v27
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v33
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v102.l, v21.h, v49.l, s56
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v54
; GFX11-TRUE16-NEXT: v_and_b32_e32 v87, 0xffff0000, v23
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.l, v9.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s29
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v7.l, s43
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v119.l, v25.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s22, 0, v70
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s44, v98, v98
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s27
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.l, v27.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s21, 0, v69
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s23, 0, v71
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v86, v86
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v87, v87
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.l, v7.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v85
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v129.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v131.l, v23.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v119, 16, v119
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v48.l, v6.h, v22.h, s44
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.l, v26.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v81
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.l, v7.h, v23.h, s40
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v87
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v97
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v129, 16, v129
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v131
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s29, v85, v119
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v14.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v65.l, v22.h, v48.l, s45
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0x8000, v11.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v83
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s25, 0, v81
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s21, s5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v64.l, v23.h, v39.l, s41
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s41, v87, v129
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s43, v97, v131
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s29
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v29.l, v13.l, s5
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v12.l
-; GFX11-TRUE16-NEXT: s_and_b32 s3, s19, s3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v98.l, v48.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v132.l, v65.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0x8000, v10.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s27, 0, v83
-; GFX11-TRUE16-NEXT: s_and_b32 s9, s25, s9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v30.l, v14.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s41
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v7.l, s43
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.l, v25.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v27.l, v11.l, s9
-; GFX11-TRUE16-NEXT: s_and_b32 s7, s23, s7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v98
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v132
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v11.h, v27.h, s14
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v32.l
-; GFX11-TRUE16-NEXT: s_and_b32 s11, s27, s11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v28.l, v12.l, s7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v87.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v97.l, v23.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v85
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v26.l, v10.l, s11
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s44, v98, v132
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.l, v27.h, v35.l, s15
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s18, s2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0x8000, v9.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v87
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v97
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s29, 0, v85
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v49.l, v32.l, s2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v102, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v22.l, s46
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v65.l, v65.l, v48.l, s44
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0x8000, v8.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0x8000, v7.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s41, 0, v87
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s43, 0, v97
-; GFX11-TRUE16-NEXT: s_and_b32 s13, s29, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v98.l, v65.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v25.l, v9.l, s13
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v21
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s15, s41, s15
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v33.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v24.l, v8.l, s15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v98
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.l, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v114.l, v52.l
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s20, s4
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v35.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v24
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v50.l, v33.l, s4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v80
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v114
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v36.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v84.l, v37.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v38.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v39.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s24, v80, v114
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v116.l, v53.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v118.l, v54.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v128.l, v55.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v130.l, v64.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.l, v52.l, v35.l, s24
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v82
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v84
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.l, v52.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v116
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v118
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v128
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v130
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v80
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s26, v82, v116
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s28, v84, v118
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s40, v86, v128
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s42, v96, v130
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s24, 0, v80
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v53.l, v53.l, v36.l, s26
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.l, v54.l, v37.l, s28
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v55.l, v55.l, v38.l, s40
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v64.l, v64.l, v39.l, s42
-; GFX11-TRUE16-NEXT: s_and_b32 s8, s24, s8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v82.l, v53.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v52.l, v35.l, s8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v84.l, v54.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v55.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v96.l, v64.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v34.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v82
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v84
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96
-; GFX11-TRUE16-NEXT: s_and_b32 s6, s22, s6
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0x8000, v36.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0x8000, v37.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0x8000, v38.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0x8000, v39.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s26, 0, v82
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s28, 0, v84
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s40, 0, v86
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s42, 0, v96
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v51.l, v34.l, s6
-; GFX11-TRUE16-NEXT: s_and_b32 s10, s26, s10
-; GFX11-TRUE16-NEXT: s_and_b32 s12, s28, s12
-; GFX11-TRUE16-NEXT: s_and_b32 s14, s40, s14
-; GFX11-TRUE16-NEXT: s_and_b32 s16, s42, s16
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v53.l, v36.l, s10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v54.l, v37.l, s12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v55.l, v38.l, s14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v64.l, v39.l, s16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v23
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v98, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v22
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v86, v86
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v96, v96
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s45, v100, v100
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v32.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s28
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v26
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s28, v80, v130
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v53.l, v65.l, v34.l, s24
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v29
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s20, 0, v69
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v5
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s27, v70, v129
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v30
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v82.l, v25.h, v37.l, s22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v49
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v102
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s22, v55, v116
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v87, v87
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v97, v97
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s43, v98, v98
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s46, v101, v101
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.l, v7.h, v23.h, s29
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v23.l, s41
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v22.l, s45
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v13.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v135, 16, v24
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s29, v81, v131
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v53
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s23, 0, v80
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s20, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v21
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s57, v112, v112
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s21, 0, v70
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v51.l, v32.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s56, v103, v52
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.l, v54.l, v33.l, s22
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v99, 0xffff0000, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v48.l, v6.h, v22.h, s43
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v14.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v34.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v38
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v7.l, s42
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, s46
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v134, 16, v85
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s43, v87, v135
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s29
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s24, 0, v81
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s23, s4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s58, v113, v113
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v28.l, s11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v21.l, s57
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v29.l, v13.l, s4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v96.l, v23.h, v39.l, s40
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v133, 16, v25
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v55.l, v71.l, v36.l, s28
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v71, 16, v52
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s44, v99, v99
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v145, 16, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v147, 16, v22
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s42, v86, v134
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s43
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v26
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s21, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s24, s5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v67, 16, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s58
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v30.l, v14.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v53.l, v34.l, s5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v33.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v39
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v144, 16, v96
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s41, v84, v133
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s27
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s22, 0, v71
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0x8000, v10.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v99.l, v22.h, v48.l, s44
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s45, v98, v145
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s47, v101, v147
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v24
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s29, 0, v86
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v21
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s44, v97, v144
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s41
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v27
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s22, s3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0x8000, v8.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v48
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v146, 16, v99
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v7.l, s45
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, s47
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s43, 0, v98
+; GFX11-TRUE16-NEXT: s_and_b32 s10, s29, s10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v52.l, v33.l, s3
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s57, v112, v114
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v26.l, v10.l, s10
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v11.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v66.l, v96.l, v39.l, s44
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v25
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s27, 0, v84
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s46, v100, v146
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.l, v68.l, v35.l, s26
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v68.l, v102.l, v49.l, s56
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v102, 16, v22
+; GFX11-TRUE16-NEXT: s_and_b32 s14, s43, s14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v50.l, v4.h, v20.h, s59
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s57
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v24.l, v8.l, s14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s12
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0x8000, v9.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s41, 0, v96
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s27, s8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0x8000, v7.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 0x8000, v6.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s45, 0, v100
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s47, 0, v102
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v27.l, v11.l, s8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v113.l, v20.h, v50.l, s60
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v21
+; GFX11-TRUE16-NEXT: s_and_b32 s12, s41, s12
+; GFX11-TRUE16-NEXT: s_and_b32 s16, s45, s16
+; GFX11-TRUE16-NEXT: s_and_b32 s18, s47, s18
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v25.l, v9.l, s12
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s57, 0, v112
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v23.l, v7.l, s16
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v22.l, v6.l, s18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v113
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v50
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v119, 16, v28
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v37
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v82
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v68
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v12.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s25, v67, v119
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 0x8000, v49.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s40, v83, v132
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s56, 0, v103
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v54
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s25
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v35.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v64.l, v82.l, v37.l, s40
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v65.l, v85.l, v38.l, s42
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s26, 0, v83
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v28
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v55
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v67.l, v99.l, v48.l, s46
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0x8000, v36.l
+; GFX11-TRUE16-NEXT: s_and_b32 s7, s26, s7
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s25, 0, v82
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v54.l, v35.l, s7
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s28, 0, v85
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v64
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v65
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s25, s6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v66
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v28.l, v12.l, s6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v67
+; GFX11-TRUE16-NEXT: s_and_b32 s9, s28, s9
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0x8000, v37.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0x8000, v38.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0x8000, v39.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0x8000, v48.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s40, 0, v87
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s42, 0, v97
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s44, 0, v99
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s46, 0, v101
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v55.l, v36.l, s9
+; GFX11-TRUE16-NEXT: s_and_b32 s11, s40, s11
+; GFX11-TRUE16-NEXT: s_and_b32 s13, s42, s13
+; GFX11-TRUE16-NEXT: s_and_b32 s15, s44, s15
+; GFX11-TRUE16-NEXT: s_and_b32 s17, s46, s17
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v64.l, v37.l, s11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v65.l, v38.l, s13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v66.l, v39.l, s15
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v67.l, v48.l, s17
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v31
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v31
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v66.l, v15.h, v31.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v67, v67
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v68, v68
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v66.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v15.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v67.l, v31.h, v66.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.l, v31.l, v15.l, s1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v68
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v69
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v31
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v31
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v15.h, v31.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v67.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v31.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v70
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v71, 16, v71
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v68, v70
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v69, v71
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v67.l, v66.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v31.l, v15.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v66.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v15.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v30.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v28
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v69, v69
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v70, v70
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v31.h, v32.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v31.l, v15.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v32
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v29
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v30
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v26
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v6.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v31, v34
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v33, v51
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v29.l, v32.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v30.l, v15.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v32.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v15.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v26
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v27
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v24
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l
; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s0
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v29.l, v66.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v102, v102
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v30.l, v15.l, s1
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s43, s17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v23.l, v7.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v5.h, v21.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v25, v25
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v22.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v48.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v23.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v21.h, v23.l, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v26, v26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21
-; GFX11-TRUE16-NEXT: s_and_b32 s4, vcc_lo, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v24.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v27, v25
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v21.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v26, v26
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v28
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v29
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v28, 0xffff0000, v20
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v25, v25
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, s2
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v27, v26
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v21.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v4.h, v20.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v28, v28
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v22.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v23.l, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v20.h, v25.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v24.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v23.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v27, v29
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v25.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v28
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v30
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v28
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v21.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v65.l, v48.l, s4
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s3, v29, v27
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v28
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v25.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v22.l, v6.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v24.l, v23.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v20
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v28, v28
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v26.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v27
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v5.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v20.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v24, v24
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v26.l, v32.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v27.l, v15.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s57, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v20
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v25, v25
; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v19
-; GFX11-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v4.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v3.h, v19.h, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v20.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v113.l, v50.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v3.h, v19.h, s1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v25.l
; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v21.l, v5.l, s3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v4.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v22
; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v19.h, v22.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v3
-; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v22.l
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v19
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v24, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v28
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v29
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v19.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v4.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v26.l, v25.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v24, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v20.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v22.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v3.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v2.h, v18.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v24
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v21.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v18.h, v23.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v28
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v27, v26
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v23.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v25
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v22.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v27
-; GFX11-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v19.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v26, v25
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v50.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v3
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s56, s19
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v68.l, v49.l, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v20
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v19
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v4
+; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v27, v26
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v19.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v28, v25
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v22.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v24.l, v50.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v18
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v4.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v19
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v2.h, v18.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v20
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v24
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v26, v25
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v22.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v18.h, v27.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v4.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v27
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v2
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v21.l, v22.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v27, v27
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v19
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v25, v23
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v26, v26
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v28
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v24.l, v23.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v21
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v24.l, v27.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v18.l, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v25
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v21.l
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v24, v24
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v17
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v21
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v27.l
; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v22
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v1.h, v17.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v24, v24
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v18.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v25
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v23.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v17.h, v22.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v27
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v22.l
-; GFX11-TRUE16-NEXT: s_and_b32 s8, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v24.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s3
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v26, v25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v21.l, v23.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v28
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v1.h, v17.h, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v17.h, v24.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v25
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v21.l, v27.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v26, v26
; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v17
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v23, v21
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v17.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v26, v26
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v16
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v21, v21
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v16
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v0.h, v16.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v16.l, s2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v25, v25
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v1.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v16.h, v21.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v21.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s1, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v17.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v17
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v21, v21
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v16
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v16
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v23, v23
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v0.h, v16.h, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v21, v21
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v16.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v1.l, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v16.h, v23.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v23
; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v0.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v22.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v23.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v16.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v18.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v27, v25
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v24.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v16
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v24.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v27, v26
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v29, v28
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v31, v30
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v21
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v31, v30
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v25
; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v21.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v22.l, v23.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v24.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v0.l, s2
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v26
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v17.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v25
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v16.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v22.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX11-TRUE16-NEXT: s_and_b32 s9, s0, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v26
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v1.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v25
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v21.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v16
+; GFX11-TRUE16-NEXT: s_and_b32 s9, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v26
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v23.l
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v27
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v24.l, v22.l, s9
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v25.l, v24.l, s9
; GFX11-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v23.l, v21.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v21.l, v23.l, s2
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v16.l, v0.l, s3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v17.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v18.l, v2.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v18.l, v2.l, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v19.l, v3.l, s8
; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v20.l, v4.l, s7
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -11459,652 +11071,542 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: scratch_load_b32 v31, off, s32
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v36, 16, v14
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v13
; GFX12-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v30
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v15
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v36, v36
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v29
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v38, v38
; GFX12-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v15
; GFX12-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v30
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v34, v34
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v37, v37
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v30.l, s4
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v12
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v33, v33
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v39, v39
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v13.h, v29.h, s6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v29
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v10
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v26
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v6
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v32, v32
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v35, v35
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v14.h, v30.h, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v13
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v22
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v50, v50
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v50.l, v29.h, v33.l, s7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v36, 16, v14
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v34, v34
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v29
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v12
; GFX12-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v28
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v99, 0xffff0000, v22
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v49, v49
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v66, v66
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v67, v67
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s46, v100, v100
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.l, v30.h, v32.l, s3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v67.l, v14.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v100.l, v30.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v28
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v48, v48
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v52, v52
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v68, v68
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v101, v101
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v68.l, v33.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v101.l, v50.l
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v51, v51
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s45, v99, v99
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v12.h, v28.h, s10
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.l, v10.h, v26.h, s18
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v66.l, v32.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v99.l, v49.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v67, 16, v67
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v100
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v11
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v53, v53
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v29.l, s8
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v28.l, s12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v68
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v101
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v9
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.l, v28.h, v34.l, s11
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v53.l, v26.h, v36.l, s19
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v66, 16, v66
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v99
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s19, v67, v100
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v27
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v26
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v25
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v64, v64
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v26.l, s20
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s9
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s13
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s20, v68, v101
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v70, v70
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v70.l, v34.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v112.l, v51.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s18, v66, v99
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s19
; GFX12-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v11
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v65, v65
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v69, v69
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v71, v71
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v27.l, s16
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v69.l, v13.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v71.l, v12.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v103.l, v29.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v113.l, v28.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v10
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v35, v35
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v48, v48
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v50.l, v50.l, v33.l, s20
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v70
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v112
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.l, v49.l, v32.l, s18
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v67.l, v30.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v14.h, v30.h, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v30
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v36, v36
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v12
; GFX12-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v27
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v26
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v114, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v49, v49
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v50, v50
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v51, v51
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v54, v54
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v68, v68
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v29.l, s7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.l, v30.h, v32.l, s2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v11
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v37, v37
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v30.l, s3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v29
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v27
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v26
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v25
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.l, v9.h, v25.h, s22
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v69
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v71, 16, v71
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v103
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v113
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v68.l, v50.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s22, v70, v112
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v66.l, v49.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v67, 16, v67
; GFX12-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v8
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v52, v52
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v55, v55
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v66, v66
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v69, v69
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s59, v114, v114
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v12.h, v28.h, s9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v11.h, v27.h, s13
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v26.l, s19
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v32
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v51
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v28
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v9
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v115, 0xffff0000, v20
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v64, v64
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v15
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v102, 0xffff0000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v38, v38
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v25
; GFX12-TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v24
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v54, v54
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v80, v80
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v81, v81
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v81.l, v11.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s21
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.l, v25.h, v37.l, s23
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v115.l, v27.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s21, v69, v103
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s23, v71, v113
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v68
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v7
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v65, v65
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v67, v67
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v80, v80
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v82, v82
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.l, v51.l, v34.l, s22
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v66, 16, v66
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s19, 0, v67
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v82, v82
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v83, v83
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v25.l, s24
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v83.l, v10.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v81
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v117.l, v26.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v115
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s21
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s23
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s20, 0, v68
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.l, v10.h, v26.h, s17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v13
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v65.l, v28.h, v34.l, s10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v68.l, v27.h, v35.l, s14
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s20
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v29
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s20, v52, v114
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v53, v53
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v70, v70
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s60, v115, v115
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v27.l, s15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v30
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v103, 0xffff0000, v21
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v33, v33
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v39, v39
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s47, v102, v102
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v13.h, v29.h, s5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v24
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v23
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v84, v84
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v96, v96
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v70.l, v51.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s18, 0, v66
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v71, v71
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v81, v81
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v83, v83
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v84, v84
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v25.l, s23
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.l, v8.h, v24.h, s25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v66, 16, v34
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v35
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v71.l, v26.h, v36.l, s18
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v65
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v68
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s23, v64, v117
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.l, v8.h, v24.h, s26
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s25
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v83
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v117
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s25, v81, v115
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v69.l, v29.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v71.l, v28.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.l, v51.l, v32.l, s20
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.l, v9.h, v25.h, s21
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s16
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s21, v53, v115
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s56, v103, v103
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.l, v5.h, v21.h, s47
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.l, v29.h, v33.l, s6
; GFX12-TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v7
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v85, v85
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s43, v97, v97
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v24.l, s28
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v23.l, s42
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v70
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v98, 0xffff0000, v6
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v55, v55
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v55.l, v24.h, v38.l, s27
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s27, v83, v117
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v6
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v85, v85
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v24.l, s27
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v36
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s24
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v85.l, v24.h, v38.l, s26
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v71
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s24, v66, v118
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s26, v69, v128
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v51
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v129, 16, v27
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s25
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v69
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v71, 16, v71
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v33
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v102.l, v21.h, v49.l, s56
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v54
; GFX12-TRUE16-NEXT: v_and_b32_e32 v87, 0xffff0000, v23
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v85.l, v9.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s29
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v7.l, s43
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v119.l, v25.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s22, 0, v70
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s44, v98, v98
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s27
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v81.l, v27.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s21, 0, v69
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s23, 0, v71
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v86, v86
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v87, v87
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v87.l, v8.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v97.l, v7.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v85
-; GFX12-TRUE16-NEXT: v_mov_b16_e64 v129.l, v24.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e64 v131.l, v23.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v119, 16, v119
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v23
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v98, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v22
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v86, v86
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v96, v96
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s45, v100, v100
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v32.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s28
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v26
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s28, v80, v130
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v53.l, v65.l, v34.l, s24
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v29
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s20, 0, v69
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v5
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s27, v70, v129
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v30
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v82.l, v25.h, v37.l, s22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v49
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v102
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s22, v55, v116
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v87, v87
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v97, v97
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s43, v98, v98
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s46, v101, v101
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v48.l, v6.h, v22.h, s44
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v13.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v83.l, v26.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v81
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.l, v7.h, v23.h, s40
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v87
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v97
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v129, 16, v129
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v131
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s29, v85, v119
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v14.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v65.l, v22.h, v48.l, s45
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0x8000, v11.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v83
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s25, 0, v81
-; GFX12-TRUE16-NEXT: s_and_b32 s5, s21, s5
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v64.l, v23.h, v39.l, s41
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s41, v87, v129
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s43, v97, v131
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s29
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.l, v7.h, v23.h, s29
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v23.l, s41
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v22.l, s45
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v13.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v135, 16, v24
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s29, v81, v131
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v53
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s23, 0, v80
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s20, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v21
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s57, v112, v112
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s21, 0, v70
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v29.l, v13.l, s5
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v12.l
-; GFX12-TRUE16-NEXT: s_and_b32 s3, s19, s3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v98.l, v48.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e64 v132.l, v65.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0x8000, v10.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s27, 0, v83
-; GFX12-TRUE16-NEXT: s_and_b32 s9, s25, s9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v51.l, v32.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s56, v103, v52
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.l, v54.l, v33.l, s22
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v99, 0xffff0000, v22
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v48.l, v6.h, v22.h, s43
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v14.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v34.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v38
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v7.l, s42
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, s46
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v134, 16, v85
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s43, v87, v135
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s29
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s24, 0, v81
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s23, s4
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s58, v113, v113
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v28.l, s11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v21.l, s57
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v30.l, v14.l, s3
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s41
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v7.l, s43
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v85.l, v25.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v27.l, v11.l, s9
-; GFX12-TRUE16-NEXT: s_and_b32 s7, s23, s7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v98
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v132
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v11.h, v27.h, s14
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v32.l
-; GFX12-TRUE16-NEXT: s_and_b32 s11, s27, s11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v29.l, v13.l, s4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v96.l, v23.h, v39.l, s40
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v133, 16, v25
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v55.l, v71.l, v36.l, s28
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v71, 16, v52
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s44, v99, v99
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v145, 16, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v147, 16, v22
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s42, v86, v134
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s43
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v26
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s21, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s5, s24, s5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v67, 16, v12
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s58
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v28.l, v12.l, s7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v87.l, v24.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v97.l, v23.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v85
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v26.l, v10.l, s11
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s44, v98, v132
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.l, v27.h, v35.l, s15
-; GFX12-TRUE16-NEXT: s_and_b32 s2, s18, s2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0x8000, v9.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v87
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v97
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s29, 0, v85
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v30.l, v14.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v53.l, v34.l, s5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v33.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v39
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v144, 16, v96
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s41, v84, v133
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s27
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s22, 0, v71
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0x8000, v10.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v99.l, v22.h, v48.l, s44
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s45, v98, v145
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s47, v101, v147
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v24
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s29, 0, v86
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v21
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s44, v97, v144
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s41
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v27
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s22, s3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0x8000, v8.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v48
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v146, 16, v99
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v7.l, s45
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, s47
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s43, 0, v98
+; GFX12-TRUE16-NEXT: s_and_b32 s10, s29, s10
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v49.l, v32.l, s2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v102, 0xffff0000, v5
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v22.l, s46
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v65.l, v65.l, v48.l, s44
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0x8000, v8.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0x8000, v7.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s41, 0, v87
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s43, 0, v97
-; GFX12-TRUE16-NEXT: s_and_b32 s13, s29, s13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v98.l, v65.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v52.l, v33.l, s3
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s57, v112, v114
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v26.l, v10.l, s10
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v11.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v66.l, v96.l, v39.l, s44
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v25
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s27, 0, v84
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s46, v100, v146
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.l, v68.l, v35.l, s26
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v68.l, v102.l, v49.l, s56
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v102, 16, v22
+; GFX12-TRUE16-NEXT: s_and_b32 s14, s43, s14
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v50.l, v4.h, v20.h, s59
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s57
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v25.l, v9.l, s13
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v21
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s15, s41, s15
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v33.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v24.l, v8.l, s14
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s12
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0x8000, v9.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s41, 0, v96
+; GFX12-TRUE16-NEXT: s_and_b32 s8, s27, s8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0x8000, v7.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 0x8000, v6.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s45, 0, v100
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s47, 0, v102
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v24.l, v8.l, s15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v98
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v80.l, v35.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v114.l, v52.l
-; GFX12-TRUE16-NEXT: s_and_b32 s4, s20, s4
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v35.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v24
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v27.l, v11.l, s8
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v113.l, v20.h, v50.l, s60
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v21
+; GFX12-TRUE16-NEXT: s_and_b32 s12, s41, s12
+; GFX12-TRUE16-NEXT: s_and_b32 s16, s45, s16
+; GFX12-TRUE16-NEXT: s_and_b32 s18, s47, s18
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v50.l, v33.l, s4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v80
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v114
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v82.l, v36.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v84.l, v37.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v86.l, v38.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v96.l, v39.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s24, v80, v114
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v116.l, v53.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v118.l, v54.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e64 v128.l, v55.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e64 v130.l, v64.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.l, v52.l, v35.l, s24
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v82
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v84
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v80.l, v52.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v116
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v118
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v128
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v130
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v80
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s26, v82, v116
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s28, v84, v118
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s40, v86, v128
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s42, v96, v130
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s24, 0, v80
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v25.l, v9.l, s12
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s57, 0, v112
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v23.l, v7.l, s16
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v22.l, v6.l, s18
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v113
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v50
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v119, 16, v28
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v37
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v82
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v68
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v12.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s25, v67, v119
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 0x8000, v49.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s40, v83, v132
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s56, 0, v103
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v54
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v53.l, v53.l, v36.l, s26
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.l, v54.l, v37.l, s28
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v55.l, v55.l, v38.l, s40
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v64.l, v64.l, v39.l, s42
-; GFX12-TRUE16-NEXT: s_and_b32 s8, s24, s8
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v82.l, v53.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s25
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v35.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v64.l, v82.l, v37.l, s40
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v65.l, v85.l, v38.l, s42
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s26, 0, v83
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v28
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v55
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v67.l, v99.l, v48.l, s46
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0x8000, v36.l
+; GFX12-TRUE16-NEXT: s_and_b32 s7, s26, s7
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s25, 0, v82
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v52.l, v35.l, s8
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v84.l, v54.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v86.l, v55.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v96.l, v64.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v34.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v82
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v84
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v86
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v96
-; GFX12-TRUE16-NEXT: s_and_b32 s6, s22, s6
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0x8000, v36.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0x8000, v37.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0x8000, v38.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0x8000, v39.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s26, 0, v82
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s28, 0, v84
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s40, 0, v86
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s42, 0, v96
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v54.l, v35.l, s7
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s28, 0, v85
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v64
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v65
+; GFX12-TRUE16-NEXT: s_and_b32 s6, s25, s6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v66
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v51.l, v34.l, s6
-; GFX12-TRUE16-NEXT: s_and_b32 s10, s26, s10
-; GFX12-TRUE16-NEXT: s_and_b32 s12, s28, s12
-; GFX12-TRUE16-NEXT: s_and_b32 s14, s40, s14
-; GFX12-TRUE16-NEXT: s_and_b32 s16, s42, s16
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v28.l, v12.l, s6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v67
+; GFX12-TRUE16-NEXT: s_and_b32 s9, s28, s9
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0x8000, v37.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0x8000, v38.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0x8000, v39.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0x8000, v48.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s40, 0, v87
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s42, 0, v97
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s44, 0, v99
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s46, 0, v101
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v53.l, v36.l, s10
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v54.l, v37.l, s12
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v55.l, v38.l, s14
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v64.l, v39.l, s16
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v55.l, v36.l, s9
+; GFX12-TRUE16-NEXT: s_and_b32 s11, s40, s11
+; GFX12-TRUE16-NEXT: s_and_b32 s13, s42, s13
+; GFX12-TRUE16-NEXT: s_and_b32 s15, s44, s15
+; GFX12-TRUE16-NEXT: s_and_b32 s17, s46, s17
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v64.l, v37.l, s11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v65.l, v38.l, s13
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v66.l, v39.l, s15
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v67.l, v48.l, s17
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v31
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v31
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v66.l, v15.h, v31.h, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v67, v67
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v68, v68
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v68.l, v66.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v69.l, v15.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v31
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v31
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v15.h, v31.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v69, v69
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v70, v70
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v15
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v31.h, v32.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v67.l, v31.h, v66.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.l, v31.l, v15.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v68
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v69
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v70.l, v67.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v71.l, v31.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v70
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v71, 16, v71
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v31.l, v15.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v32
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v29
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v30
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v68, v70
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v69, v71
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v31, v34
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v33, v51
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v29.l, v32.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v30.l, v15.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v32.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v15.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v26
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v27
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v67.l, v66.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v31.l, v15.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v66.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v15.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v29.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v30.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v27
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v28
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v26
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v27
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v6.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v24
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l
; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s0
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s3, s1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v29.l, v66.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v102, v102
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v30.l, v15.l, s1
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s43, s17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v23.l, v7.l, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v5.h, v21.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v25, v25
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v22.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v48.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v29.l, v23.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v21.h, v23.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v26, v26
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21
-; GFX12-TRUE16-NEXT: s_and_b32 s4, vcc_lo, s1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v24.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v27, v25
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v21.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v26, v26
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v28
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v29
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v28, 0xffff0000, v20
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v25, v25
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, s2
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v27, v26
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v5.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v21.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v4.h, v20.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v28, v28
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v22.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v23.l, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v26
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v20.h, v25.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v30.l, v24.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v23.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v27, v29
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v26.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v29.l, v25.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v28
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v30
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v26.l, v32.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v27.l, v15.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s57, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v20
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v25, v25
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v19
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v20.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v28
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v21.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v65.l, v48.l, s4
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s3, v29, v27
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v28
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v4
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v25.l, s3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v22.l, v6.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v24.l, v23.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v20
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v28, v28
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v26.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v27
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v5.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v113.l, v50.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v20.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v24, v24
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v19
-; GFX12-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v4.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v3.h, v19.h, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v3.h, v19.h, s1
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v25.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v21.l, v5.l, s3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v23.l, v20.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v4.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v19.h, v22.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v3
-; GFX12-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v24
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v21.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v29.l, v22.l
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v19
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v24, v23
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v28
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v29
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v19.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v22
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v4.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v19.h, v22.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v50.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v3
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s56, s19
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v26.l, v25.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v24, v23
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v68.l, v49.l, s2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v20
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v19
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v4
+; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v27, v26
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v19.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v28, v25
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v18
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v24.l, v20.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v22.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v24.l, v50.l, s2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v22.l, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v19.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v3.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v18
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v4.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v19
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v2.h, v18.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v24
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v21.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v2.h, v18.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v20
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v24
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v26, v25
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v22.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v18.h, v23.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v25
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v28
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v27, v26
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v24.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v23.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v25
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v22.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v18.h, v27.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v4.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v26
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v27
-; GFX12-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v19.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v26, v25
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, s2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v27
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v2
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX12-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v21.l, v22.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v27, v27
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v19
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v25, v23
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v26, v26
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v28
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v21
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v24.l, v23.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v24.l, v27.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v18.l, s0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v25
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v22.l, v21.l
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v24, v24
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v17
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v21
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v27.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v22
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v1.h, v17.h, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v24, v24
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v18.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v25
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v23.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v17.h, v22.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v26
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v27
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v22.l
-; GFX12-TRUE16-NEXT: s_and_b32 s8, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v24.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s3
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v26, v25
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v1.h, v17.h, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v18
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v17.h, v24.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v25
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v21.l, v23.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v27
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v28
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v21.l, v27.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v26, v26
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v17
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v23, v21
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v17.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v26, v26
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v16
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v21, v21
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v16
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s8, s1, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v17.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v17
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v21, v21
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v16
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v16
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v23, v23
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v0.h, v16.h, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v16.l, s2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v25, v25
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v17.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v1.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v16.h, v21.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v29.l, v21.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v0.h, v16.h, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v21, v21
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v16.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v1.l, s2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v16.h, v23.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v23
; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v31.l, v0.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v22.l, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v28.l, v23.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v30.l, v16.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v31
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v30
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v18.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v27, v25
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v24.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v18
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v16
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v24.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v27, v26
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v29, v28
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v31, v30
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v21
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v31, v30
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v25
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v21.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v22.l, v23.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v24.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v0.l, s2
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v26
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v26.l, v17.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v25
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v25.l, v23.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v27.l, v16.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v22.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX12-TRUE16-NEXT: s_and_b32 s9, s0, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v26
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v1.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v25
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v21.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v16
+; GFX12-TRUE16-NEXT: s_and_b32 s9, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v26
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v23.l
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v27
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v24.l, v22.l, s9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v25.l, v24.l, s9
; GFX12-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v23.l, v21.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v21.l, v23.l, s2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v16.l, v0.l, s3
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v17.l, v1.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v18.l, v2.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v18.l, v2.l, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v19.l, v3.l, s8
; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v20.l, v4.l, s7
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -12867,27 +12369,24 @@ define bfloat @v_minimumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
; GFX11-TRUE16-LABEL: v_minimumnum_bf16_no_ieee:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
@@ -12925,30 +12424,28 @@ define bfloat @v_minimumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -13179,17 +12676,11 @@ define <2 x bfloat> @v_minimumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v7
@@ -13198,11 +12689,8 @@ define <2 x bfloat> @v_minimumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
@@ -13279,17 +12767,11 @@ define <2 x bfloat> @v_minimumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v4, v6
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v7
@@ -13300,11 +12782,8 @@ define <2 x bfloat> @v_minimumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v2.l
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v4
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
@@ -13621,61 +13100,51 @@ define <3 x bfloat> @v_minimumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v8, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v7, v7
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v4.l, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v4.l, s3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v9
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v10, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v9
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v8, v11
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v6.l, v4.l, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, s0
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
; GFX11-TRUE16-NEXT: s_and_b32 s0, s3, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
; GFX11-TRUE16-NEXT: s_and_b32 s1, s4, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s0
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v3bf16_no_ieee:
@@ -13750,66 +13219,55 @@ define <3 x bfloat> @v_minimumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v8, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v7, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, s0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s3
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v2.h, v4.l, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v1.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v4.l, s3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v3.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v8
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v9
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v10, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v9
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v10
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v8, v11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v6.l, v4.l, s0
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
; GFX12-TRUE16-NEXT: s_and_b32 s0, s3, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
; GFX12-TRUE16-NEXT: s_and_b32 s1, s4, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s0
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v3bf16_no_ieee:
@@ -14210,75 +13668,62 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v1.h, v3.h, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v10, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v4, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v4.l, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.h, v5.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v6.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.h, v6.l, s3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v10, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v9, v12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v7
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v10
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v6.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v8, v13
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v13, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v6.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v5.l
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v8
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
; GFX11-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v4.l, s5
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v5.l, s5
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v6.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v6.l, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v4bf16_no_ieee:
@@ -14370,84 +13815,71 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v1.h, v3.h, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v10, v10
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v4, v4
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v3.h, v4.l, s0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.h, v5.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.h, v6.l, s3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v6.l, s3
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v11.l, v6.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v10, v8
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v7.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v9, v12
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v5.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v7
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v10
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v6.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v8, v13
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v13, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v9
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v6.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v5.l
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v9.l, v3.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v8
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
; GFX12-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v4.l, s5
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v5.l, s5
; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v6.l, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v6.l, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v4bf16_no_ieee:
More information about the llvm-commits
mailing list