[llvm] 0a015cc - [AMDGPU] Replace zext pattern from reg_sequence to cvt_u32_u16 (#208045)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 8 09:26:01 PDT 2026
Author: Guo Chen
Date: 2026-07-08T12:25:55-04:00
New Revision: 0a015cc516abb7f54bcdd94763f44219b6298bd9
URL: https://github.com/llvm/llvm-project/commit/0a015cc516abb7f54bcdd94763f44219b6298bd9
DIFF: https://github.com/llvm/llvm-project/commit/0a015cc516abb7f54bcdd94763f44219b6298bd9.diff
LOG: [AMDGPU] Replace zext pattern from reg_sequence to cvt_u32_u16 (#208045)
Isel pattern putting imm inside reg_sequence create side effects when
register coalescer join these `copy` from imm
```
%1 = v_mov_b16_t16_e64 ...
%2.hi16 = copy %1
%2.lo16 = ....
...
%3.hi16 = copy %1
%3.lo16 = ....
....
%4.hi16 = copy %1
%4.lo16 = ....
```
to
```
%1 = v_mov_b16_t16_e64 ...
%2.hi16 = copy %1
%2.lo16 = ....
...
%2.lo16 = ... (reuse %2 and repeat)
....
```
When the number of copy increase this inserts a large number of WAR
hazzards on the reused virutal reg, and the machine scheduler bail out
getting higher reg pressure after sorting.
Beside this, replacing with cvt_u32_u16 provides better code quality
Added:
Modified:
llvm/lib/Target/AMDGPU/SIInstructions.td
llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-add.s16.mir
llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-anyext.mir
llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.s16.mir
llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.s16.mir
llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.s16.mir
llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll
llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll
llvm/test/CodeGen/AMDGPU/atomics-system-scope.ll
llvm/test/CodeGen/AMDGPU/bf16.ll
llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
llvm/test/CodeGen/AMDGPU/clamp-modifier.ll
llvm/test/CodeGen/AMDGPU/ctls.ll
llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
llvm/test/CodeGen/AMDGPU/fold-int-pow2-with-fmul-or-fdiv.ll
llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
llvm/test/CodeGen/AMDGPU/idot4u.ll
llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll
llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
llvm/test/CodeGen/AMDGPU/mad.u16.ll
llvm/test/CodeGen/AMDGPU/preserve-hi16.ll
llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
llvm/test/CodeGen/AMDGPU/srl-bitcast-bv.ll
llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll
llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 49441b86d59eb..d3ea76ebb647d 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -3544,17 +3544,19 @@ def : GCNPat <
let True16Predicate = UseRealTrue16Insts in {
def : GCNPat<
(i32 (DivergentUnaryFrag<zext> i16:$src)),
- (REG_SEQUENCE VGPR_32, VGPR_16:$src, lo16, (V_MOV_B16_t16_e64 0, (i16 0), 0), hi16)
+ (i32 (V_CVT_U32_U16_t16_e64 SRCMODS.NONE, $src))
>;
def : GCNPat<
(i64 (DivergentUnaryFrag<zext> i16:$src)),
- (REG_SEQUENCE VReg_64, $src, lo16, (V_MOV_B16_t16_e64 0, (i16 0), 0), hi16, (V_MOV_B32_e32 (i32 0)), sub1)
+ (REG_SEQUENCE VReg_64,
+ (V_CVT_U32_U16_t16_e64 SRCMODS.NONE, $src), sub0,
+ (S_MOV_B32 (i32 0)), sub1)
>;
def : GCNPat<
(i32 (zext (i16 (bitconvert fp16_zeros_high_16bits:$src)))),
- (REG_SEQUENCE VGPR_32, VGPR_16:$src, lo16, (V_MOV_B16_t16_e64 0, (i16 0), 0), hi16)
+ (i32 (V_CVT_U32_U16_t16_e64 SRCMODS.NONE, $src))
>;
def : GCNPat <
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll
index 3a1977f69f71c..ed5763422715a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll
@@ -501,8 +501,8 @@ define amdgpu_ps float @v_andn2_i16_sv(i16 inreg %src0, i16 %src1) {
; GFX11-TRUE16-LABEL: v_andn2_i16_sv:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, -1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, s2, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
; GFX11-FAKE16-LABEL: v_andn2_i16_sv:
@@ -536,8 +536,8 @@ define amdgpu_ps float @v_andn2_i16_vs(i16 %src0, i16 inreg %src1) {
; GFX11-TRUE16-LABEL: v_andn2_i16_vs:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_xor_b32 s0, s2, -1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
; GFX11-FAKE16-LABEL: v_andn2_i16_vs:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-add.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-add.s16.mir
index 563adaf6cc9d0..c595042284883 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-add.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-add.s16.mir
@@ -95,9 +95,8 @@ body: |
; GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
; GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
; GFX11-TRUE16-NEXT: [[V_ADD_NC_U16_t16_e64_:%[0-9]+]]:vgpr_16 = V_ADD_NC_U16_t16_e64 0, [[COPY2]], 0, [[COPY3]], 0, 0, implicit $exec
- ; GFX11-TRUE16-NEXT: [[V_MOV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_MOV_B16_t16_e64 0, 0, 0, implicit $exec
- ; GFX11-TRUE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_ADD_NC_U16_t16_e64_]], %subreg.lo16, [[V_MOV_B16_t16_e64_]], %subreg.hi16
- ; GFX11-TRUE16-NEXT: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
+ ; GFX11-TRUE16-NEXT: [[V_CVT_U32_U16_t16_e64_:%[0-9]+]]:vgpr_32 = V_CVT_U32_U16_t16_e64 0, [[V_ADD_NC_U16_t16_e64_]], 0, implicit $exec
+ ; GFX11-TRUE16-NEXT: S_ENDPGM 0, implicit [[V_CVT_U32_U16_t16_e64_]]
;
; GFX11-FAKE16-LABEL: name: add_s16_zext_to_s32
; GFX11-FAKE16: liveins: $vgpr0, $vgpr1
@@ -196,9 +195,8 @@ body: |
; GFX11-TRUE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX11-TRUE16-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
; GFX11-TRUE16-NEXT: [[V_SUB_NC_U16_t16_e64_:%[0-9]+]]:vgpr_16 = V_SUB_NC_U16_t16_e64 0, [[COPY1]], 0, 64, 0, 0, implicit $exec
- ; GFX11-TRUE16-NEXT: [[V_MOV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_MOV_B16_t16_e64 0, 0, 0, implicit $exec
- ; GFX11-TRUE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_SUB_NC_U16_t16_e64_]], %subreg.lo16, [[V_MOV_B16_t16_e64_]], %subreg.hi16
- ; GFX11-TRUE16-NEXT: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
+ ; GFX11-TRUE16-NEXT: [[V_CVT_U32_U16_t16_e64_:%[0-9]+]]:vgpr_32 = V_CVT_U32_U16_t16_e64 0, [[V_SUB_NC_U16_t16_e64_]], 0, implicit $exec
+ ; GFX11-TRUE16-NEXT: S_ENDPGM 0, implicit [[V_CVT_U32_U16_t16_e64_]]
;
; GFX11-FAKE16-LABEL: name: add_s16_neg_inline_const_64_zext_to_s32
; GFX11-FAKE16: liveins: $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-anyext.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-anyext.mir
index 6703a1608a7a2..2188ef8a0ee25 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-anyext.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-anyext.mir
@@ -252,9 +252,8 @@ body: |
; GFX11-TRUE16-NEXT: {{ $}}
; GFX11-TRUE16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX11-TRUE16-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX11-TRUE16-NEXT: [[V_MOV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_MOV_B16_t16_e64 0, 0, 0, implicit $exec
- ; GFX11-TRUE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY1]], %subreg.lo16, [[V_MOV_B16_t16_e64_]], %subreg.hi16
- ; GFX11-TRUE16-NEXT: $vgpr0 = COPY [[REG_SEQUENCE]]
+ ; GFX11-TRUE16-NEXT: [[V_CVT_U32_U16_t16_e64_:%[0-9]+]]:vgpr_32 = V_CVT_U32_U16_t16_e64 0, [[COPY1]], 0, implicit $exec
+ ; GFX11-TRUE16-NEXT: $vgpr0 = COPY [[V_CVT_U32_U16_t16_e64_]]
;
; GFX11-FAKE16-LABEL: name: anyext_vgpr_s1_to_vgpr_s16
; GFX11-FAKE16: liveins: $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.s16.mir
index 20e39408339a8..132082bbc5d23 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.s16.mir
@@ -285,9 +285,8 @@ body: |
; GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
; GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
; GFX11-TRUE16-NEXT: [[V_ASHRREV_I16_t16_e64_:%[0-9]+]]:vgpr_16 = V_ASHRREV_I16_t16_e64 0, [[COPY3]], 0, [[COPY2]], 0, implicit $exec
- ; GFX11-TRUE16-NEXT: [[V_MOV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_MOV_B16_t16_e64 0, 0, 0, implicit $exec
- ; GFX11-TRUE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_ASHRREV_I16_t16_e64_]], %subreg.lo16, [[V_MOV_B16_t16_e64_]], %subreg.hi16
- ; GFX11-TRUE16-NEXT: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
+ ; GFX11-TRUE16-NEXT: [[V_CVT_U32_U16_t16_e64_:%[0-9]+]]:vgpr_32 = V_CVT_U32_U16_t16_e64 0, [[V_ASHRREV_I16_t16_e64_]], 0, implicit $exec
+ ; GFX11-TRUE16-NEXT: S_ENDPGM 0, implicit [[V_CVT_U32_U16_t16_e64_]]
;
; GFX11-FAKE16-LABEL: name: ashr_s16_s16_vv_zext_to_s32
; GFX11-FAKE16: liveins: $vgpr0, $vgpr1
@@ -357,9 +356,10 @@ body: |
; GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
; GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
; GFX11-TRUE16-NEXT: [[V_ASHRREV_I16_t16_e64_:%[0-9]+]]:vgpr_16 = V_ASHRREV_I16_t16_e64 0, [[COPY3]], 0, [[COPY2]], 0, implicit $exec
- ; GFX11-TRUE16-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; GFX11-TRUE16-NEXT: [[V_MOV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_MOV_B16_t16_e64 0, 0, 0, implicit $exec
- ; GFX11-TRUE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_ASHRREV_I16_t16_e64_]], %subreg.lo16, [[V_MOV_B16_t16_e64_]], %subreg.hi16, [[V_MOV_B32_e32_]], %subreg.sub1
+ ; GFX11-TRUE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX11-TRUE16-NEXT: [[V_CVT_U32_U16_t16_e64_:%[0-9]+]]:vgpr_32 = V_CVT_U32_U16_t16_e64 0, [[V_ASHRREV_I16_t16_e64_]], 0, implicit $exec
+ ; GFX11-TRUE16-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX11-TRUE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_CVT_U32_U16_t16_e64_]], %subreg.sub0, [[COPY4]], %subreg.sub1
; GFX11-TRUE16-NEXT: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
;
; GFX11-FAKE16-LABEL: name: ashr_s16_vv_zext_to_s64
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.s16.mir
index 638582cc62275..2b317af916aa8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-lshr.s16.mir
@@ -281,9 +281,8 @@ body: |
; GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
; GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
; GFX11-TRUE16-NEXT: [[V_LSHRREV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_LSHRREV_B16_t16_e64 0, [[COPY3]], 0, [[COPY2]], 0, implicit $exec
- ; GFX11-TRUE16-NEXT: [[V_MOV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_MOV_B16_t16_e64 0, 0, 0, implicit $exec
- ; GFX11-TRUE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_LSHRREV_B16_t16_e64_]], %subreg.lo16, [[V_MOV_B16_t16_e64_]], %subreg.hi16
- ; GFX11-TRUE16-NEXT: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
+ ; GFX11-TRUE16-NEXT: [[V_CVT_U32_U16_t16_e64_:%[0-9]+]]:vgpr_32 = V_CVT_U32_U16_t16_e64 0, [[V_LSHRREV_B16_t16_e64_]], 0, implicit $exec
+ ; GFX11-TRUE16-NEXT: S_ENDPGM 0, implicit [[V_CVT_U32_U16_t16_e64_]]
;
; GFX11-FAKE16-LABEL: name: lshr_s16_s16_vv_zext_to_s32
; GFX11-FAKE16: liveins: $vgpr0, $vgpr1
@@ -353,9 +352,10 @@ body: |
; GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
; GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
; GFX11-TRUE16-NEXT: [[V_LSHRREV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_LSHRREV_B16_t16_e64 0, [[COPY3]], 0, [[COPY2]], 0, implicit $exec
- ; GFX11-TRUE16-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; GFX11-TRUE16-NEXT: [[V_MOV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_MOV_B16_t16_e64 0, 0, 0, implicit $exec
- ; GFX11-TRUE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_LSHRREV_B16_t16_e64_]], %subreg.lo16, [[V_MOV_B16_t16_e64_]], %subreg.hi16, [[V_MOV_B32_e32_]], %subreg.sub1
+ ; GFX11-TRUE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX11-TRUE16-NEXT: [[V_CVT_U32_U16_t16_e64_:%[0-9]+]]:vgpr_32 = V_CVT_U32_U16_t16_e64 0, [[V_LSHRREV_B16_t16_e64_]], 0, implicit $exec
+ ; GFX11-TRUE16-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX11-TRUE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_CVT_U32_U16_t16_e64_]], %subreg.sub0, [[COPY4]], %subreg.sub1
; GFX11-TRUE16-NEXT: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
;
; GFX11-FAKE16-LABEL: name: lshr_s16_vv_zext_to_s64
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.s16.mir
index 5a2bd2e2b6cae..de8b7fe13038a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-shl.s16.mir
@@ -281,9 +281,8 @@ body: |
; GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
; GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
; GFX11-TRUE16-NEXT: [[V_LSHLREV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_LSHLREV_B16_t16_e64 0, [[COPY3]], 0, [[COPY2]], 0, implicit $exec
- ; GFX11-TRUE16-NEXT: [[V_MOV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_MOV_B16_t16_e64 0, 0, 0, implicit $exec
- ; GFX11-TRUE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_LSHLREV_B16_t16_e64_]], %subreg.lo16, [[V_MOV_B16_t16_e64_]], %subreg.hi16
- ; GFX11-TRUE16-NEXT: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
+ ; GFX11-TRUE16-NEXT: [[V_CVT_U32_U16_t16_e64_:%[0-9]+]]:vgpr_32 = V_CVT_U32_U16_t16_e64 0, [[V_LSHLREV_B16_t16_e64_]], 0, implicit $exec
+ ; GFX11-TRUE16-NEXT: S_ENDPGM 0, implicit [[V_CVT_U32_U16_t16_e64_]]
;
; GFX11-FAKE16-LABEL: name: shl_s16_s16_vv_zext_to_s32
; GFX11-FAKE16: liveins: $vgpr0, $vgpr1
@@ -353,9 +352,10 @@ body: |
; GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
; GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
; GFX11-TRUE16-NEXT: [[V_LSHLREV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_LSHLREV_B16_t16_e64 0, [[COPY3]], 0, [[COPY2]], 0, implicit $exec
- ; GFX11-TRUE16-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; GFX11-TRUE16-NEXT: [[V_MOV_B16_t16_e64_:%[0-9]+]]:vgpr_16 = V_MOV_B16_t16_e64 0, 0, 0, implicit $exec
- ; GFX11-TRUE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_LSHLREV_B16_t16_e64_]], %subreg.lo16, [[V_MOV_B16_t16_e64_]], %subreg.hi16, [[V_MOV_B32_e32_]], %subreg.sub1
+ ; GFX11-TRUE16-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GFX11-TRUE16-NEXT: [[V_CVT_U32_U16_t16_e64_:%[0-9]+]]:vgpr_32 = V_CVT_U32_U16_t16_e64 0, [[V_LSHLREV_B16_t16_e64_]], 0, implicit $exec
+ ; GFX11-TRUE16-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GFX11-TRUE16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[V_CVT_U32_U16_t16_e64_]], %subreg.sub0, [[COPY4]], %subreg.sub1
; GFX11-TRUE16-NEXT: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
;
; GFX11-FAKE16-LABEL: name: shl_s16_vv_zext_to_s64
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
index 32b790d4b986b..e1dda7317c9fe 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
@@ -189,7 +189,7 @@ define zeroext i16 @v_mul_i16_zeroext(i16 zeroext %num, i16 zeroext %den) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_mul_i16_zeroext:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll
index daefb43cc901b..e9254bd4441b2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll
@@ -501,8 +501,8 @@ define amdgpu_ps float @v_orn2_i16_sv(i16 inreg %src0, i16 %src1) {
; GFX11-TRUE16-LABEL: v_orn2_i16_sv:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, -1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, s2, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
; GFX11-FAKE16-LABEL: v_orn2_i16_sv:
@@ -536,8 +536,8 @@ define amdgpu_ps float @v_orn2_i16_vs(i16 %src0, i16 inreg %src1) {
; GFX11-TRUE16-LABEL: v_orn2_i16_vs:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_xor_b32 s0, s2, -1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
; GFX11-FAKE16-LABEL: v_orn2_i16_vs:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll
index adbb5b34c8be9..042c56ec364b2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll
@@ -617,8 +617,8 @@ define i32 @v_shl_i32_zext_i16(i16 %x) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0x3fff, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 2, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_shl_i32_zext_i16:
@@ -751,12 +751,10 @@ define <2 x i32> @v_shl_v2i32_zext_v2i16(<2 x i16> %x) {
; GFX11-TRUE16-LABEL: v_shl_v2i32_zext_v2i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0x3fff3fff, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
-; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v2, 2, v1 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3fff3fff, v0
+; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v1, 2, v0 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v1.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_shl_v2i32_zext_v2i16:
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index 75699d12c3669..cb6e860c92cc1 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -14400,15 +14400,15 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1164-TRUE16-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-TRUE16-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1164-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1164-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s9, v1
-; GFX1164-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX1164-TRUE16-NEXT: v_add_f16_e32 v0.l, s8, v0.l
; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1164-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s9, v0
+; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164-TRUE16-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX1164-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1164-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1164-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1164-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -14494,14 +14494,14 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1132-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1132-TRUE16-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1132-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
-; GFX1132-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX1132-TRUE16-NEXT: v_add_f16_e32 v0.l, s8, v0.l
; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1132-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1132-TRUE16-NEXT: v_and_or_b32 v0, v1, s3, v0
-; GFX1132-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1132-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], 0 glc
; GFX1132-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -14586,15 +14586,15 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1264-TRUE16-NEXT: s_mov_b64 s[2:3], 0
; GFX1264-TRUE16-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1264-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1264-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s9, v1
-; GFX1264-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX1264-TRUE16-NEXT: v_add_f16_e32 v0.l, s8, v0.l
; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1264-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1264-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s9, v0
+; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1264-TRUE16-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX1264-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1264-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1264-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1264-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -14680,14 +14680,14 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1232-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1232-TRUE16-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1232-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1232-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
-; GFX1232-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX1232-TRUE16-NEXT: v_add_f16_e32 v0.l, s8, v0.l
; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1232-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1232-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1232-TRUE16-NEXT: v_and_or_b32 v0, v1, s3, v0
-; GFX1232-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1232-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1232-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1232-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -14771,15 +14771,15 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1364-TRUE16-NEXT: s_mov_b64 s[2:3], 0
; GFX1364-TRUE16-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1364-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1364-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s9, v1
-; GFX1364-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX1364-TRUE16-NEXT: v_add_f16_e32 v0.l, s8, v0.l
; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1364-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1364-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s9, v0
+; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1364-TRUE16-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1364-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1364-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX1364-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1364-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -14863,14 +14863,14 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX1332-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1332-TRUE16-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX1332-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1332-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
-; GFX1332-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX1332-TRUE16-NEXT: v_add_f16_e32 v0.l, s8, v0.l
; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1332-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1332-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
+; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1332-TRUE16-NEXT: v_and_or_b32 v0, v1, s3, v0
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1332-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX1332-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], off, s[4:7], null th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX1332-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -15645,15 +15645,14 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1364-TRUE16-NEXT: s_mov_b64 s[2:3], 0
; GFX1364-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX1364-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1364-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s9, v1
-; GFX1364-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
; GFX1364-TRUE16-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s8 op_sel:[0,1,0] op_sel_hi:[1,1,1]
; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1364-TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1364-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX1364-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1364-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s9, v2
+; GFX1364-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s9, v0
; GFX1364-TRUE16-NEXT: v_and_or_b32 v0, v1, s10, v0
; GFX1364-TRUE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1364-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
@@ -15742,15 +15741,14 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX1332-TRUE16-NEXT: s_mov_b32 s6, -1
; GFX1332-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX1332-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1332-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s2, v1
-; GFX1332-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
; GFX1332-TRUE16-NEXT: v_fma_mix_f32_bf16 v0, v0, 1.0, s8 op_sel:[0,1,0] op_sel_hi:[1,1,1]
; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1332-TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1332-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX1332-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1332-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v2
+; GFX1332-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s2, v0
; GFX1332-TRUE16-NEXT: v_and_or_b32 v0, v1, s3, v0
; GFX1332-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1332-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
diff --git a/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll b/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll
index b0fae21a88484..af27d01b285c7 100644
--- a/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll
@@ -841,9 +841,10 @@ define i16 @global_atomic_usub_sat_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX11-GISEL-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-GISEL-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_sub_nc_u16 v3.l, v4.l, v2.l clamp
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_sub_nc_u16 v2.h, v4.l, v2.l clamp
+; GFX11-GISEL-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-GISEL-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -873,9 +874,10 @@ define i16 @global_atomic_usub_sat_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX12-GISEL-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-GISEL-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_sub_nc_u16 v3.l, v4.l, v2.l clamp
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_sub_nc_u16 v2.h, v4.l, v2.l clamp
+; GFX12-GISEL-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-GISEL-NEXT: s_wait_storecnt 0x0
; GFX12-GISEL-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -951,9 +953,10 @@ define i16 @global_atomic_usub_sat_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX11-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX11-SDAG-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-SDAG-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT: v_sub_nc_u16 v3.l, v4.l, v2.l clamp
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_sub_nc_u16 v2.h, v4.l, v2.l clamp
+; GFX11-SDAG-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-SDAG-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -983,9 +986,10 @@ define i16 @global_atomic_usub_sat_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX12-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX12-SDAG-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_sub_nc_u16 v3.l, v4.l, v2.l clamp
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_sub_nc_u16 v2.h, v4.l, v2.l clamp
+; GFX12-SDAG-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-SDAG-NEXT: s_wait_storecnt 0x0
; GFX12-SDAG-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -1066,9 +1070,10 @@ define i16 @global_atomic_usub_sat_offset_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX11-GISEL-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-GISEL-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_sub_nc_u16 v3.l, v4.l, v2.l clamp
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_sub_nc_u16 v2.h, v4.l, v2.l clamp
+; GFX11-GISEL-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-GISEL-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2048 glc
@@ -1098,9 +1103,10 @@ define i16 @global_atomic_usub_sat_offset_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX12-GISEL-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-GISEL-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_sub_nc_u16 v3.l, v4.l, v2.l clamp
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_sub_nc_u16 v2.h, v4.l, v2.l clamp
+; GFX12-GISEL-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-GISEL-NEXT: s_wait_storecnt 0x0
; GFX12-GISEL-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -1177,9 +1183,10 @@ define i16 @global_atomic_usub_sat_offset_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX11-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX11-SDAG-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-SDAG-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT: v_sub_nc_u16 v3.l, v4.l, v2.l clamp
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_sub_nc_u16 v2.h, v4.l, v2.l clamp
+; GFX11-SDAG-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-SDAG-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2048 glc
@@ -1209,9 +1216,10 @@ define i16 @global_atomic_usub_sat_offset_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX12-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX12-SDAG-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_sub_nc_u16 v3.l, v4.l, v2.l clamp
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_sub_nc_u16 v2.h, v4.l, v2.l clamp
+; GFX12-SDAG-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-SDAG-NEXT: s_wait_storecnt 0x0
; GFX12-SDAG-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -1289,9 +1297,9 @@ define void @global_atomic_usub_sat_nortn_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX11-GISEL-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-NEXT: v_sub_nc_u16 v3.l, v4.l, v2.l clamp
-; GFX11-GISEL-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_sub_nc_u16 v2.h, v4.l, v2.l clamp
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-GISEL-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-GISEL-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -1320,9 +1328,9 @@ define void @global_atomic_usub_sat_nortn_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX12-GISEL-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: v_sub_nc_u16 v3.l, v4.l, v2.l clamp
-; GFX12-GISEL-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_sub_nc_u16 v2.h, v4.l, v2.l clamp
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-GISEL-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-GISEL-NEXT: s_wait_storecnt 0x0
; GFX12-GISEL-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -1395,9 +1403,9 @@ define void @global_atomic_usub_sat_nortn_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX11-SDAG-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX11-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-NEXT: v_sub_nc_u16 v3.l, v4.l, v2.l clamp
-; GFX11-SDAG-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_sub_nc_u16 v2.h, v4.l, v2.l clamp
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-SDAG-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-SDAG-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -1426,9 +1434,9 @@ define void @global_atomic_usub_sat_nortn_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX12-SDAG-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX12-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-NEXT: v_sub_nc_u16 v3.l, v4.l, v2.l clamp
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_sub_nc_u16 v2.h, v4.l, v2.l clamp
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-SDAG-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-SDAG-NEXT: s_wait_storecnt 0x0
; GFX12-SDAG-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -1507,9 +1515,9 @@ define void @global_atomic_usub_sat_offset_nortn_16(ptr addrspace(1) %ptr, i16 %
; GFX11-GISEL-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-NEXT: v_sub_nc_u16 v3.l, v4.l, v2.l clamp
-; GFX11-GISEL-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_sub_nc_u16 v2.h, v4.l, v2.l clamp
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-GISEL-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-GISEL-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-GISEL-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2048 glc
@@ -1538,9 +1546,9 @@ define void @global_atomic_usub_sat_offset_nortn_16(ptr addrspace(1) %ptr, i16 %
; GFX12-GISEL-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: v_sub_nc_u16 v3.l, v4.l, v2.l clamp
-; GFX12-GISEL-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_sub_nc_u16 v2.h, v4.l, v2.l clamp
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-GISEL-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-GISEL-NEXT: s_wait_storecnt 0x0
; GFX12-GISEL-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -1615,9 +1623,9 @@ define void @global_atomic_usub_sat_offset_nortn_16(ptr addrspace(1) %ptr, i16 %
; GFX11-SDAG-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX11-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-NEXT: v_sub_nc_u16 v3.l, v4.l, v2.l clamp
-; GFX11-SDAG-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_sub_nc_u16 v2.h, v4.l, v2.l clamp
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-SDAG-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-SDAG-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2048 glc
@@ -1646,9 +1654,9 @@ define void @global_atomic_usub_sat_offset_nortn_16(ptr addrspace(1) %ptr, i16 %
; GFX12-SDAG-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX12-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-NEXT: v_sub_nc_u16 v3.l, v4.l, v2.l clamp
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_sub_nc_u16 v2.h, v4.l, v2.l clamp
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-SDAG-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-SDAG-NEXT: s_wait_storecnt 0x0
; GFX12-SDAG-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -1746,11 +1754,11 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_16(ptr addrsp
; GFX11-GISEL-NEXT: s_mov_b32 s3, 0
; GFX11-GISEL-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, v1
-; GFX11-GISEL-NEXT: v_mov_b16_e32 v1.h, 0
; GFX11-GISEL-NEXT: v_sub_nc_u16 v1.l, v2.l, s2 clamp
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_cvt_u32_u16_e32 v1, v1.l
; GFX11-GISEL-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v1
; GFX11-GISEL-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:2048 glc
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -1779,11 +1787,11 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_16(ptr addrsp
; GFX12-GISEL-NEXT: s_mov_b32 s3, 0
; GFX12-GISEL-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_mov_b32_e32 v2, v1
-; GFX12-GISEL-NEXT: v_mov_b16_e32 v1.h, 0
; GFX12-GISEL-NEXT: v_sub_nc_u16 v1.l, v2.l, s2 clamp
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_cvt_u32_u16_e32 v1, v1.l
; GFX12-GISEL-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v1
; GFX12-GISEL-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
@@ -1881,11 +1889,11 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_16(ptr addrsp
; GFX11-SDAG-NEXT: s_mov_b32 s3, 0
; GFX11-SDAG-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX11-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, v1
-; GFX11-SDAG-NEXT: v_mov_b16_e32 v1.h, 0
; GFX11-SDAG-NEXT: v_sub_nc_u16 v1.l, v2.l, s2 clamp
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_cvt_u32_u16_e32 v1, v1.l
; GFX11-SDAG-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v1
; GFX11-SDAG-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:2048 glc
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
@@ -1915,11 +1923,11 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_16(ptr addrsp
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0
; GFX12-SDAG-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX12-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, v1
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v1.h, 0
; GFX12-SDAG-NEXT: v_sub_nc_u16 v1.l, v2.l, s2 clamp
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_cvt_u32_u16_e32 v1, v1.l
; GFX12-SDAG-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v1
; GFX12-SDAG-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
@@ -2010,9 +2018,10 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_16(ptr
; GFX11-GISEL-NEXT: s_mov_b32 s3, 0
; GFX11-GISEL-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_sub_nc_u16 v0.l, v1.l, s2 clamp
-; GFX11-GISEL-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX11-GISEL-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
; GFX11-GISEL-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:2048 glc
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -2037,9 +2046,10 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_16(ptr
; GFX12-GISEL-NEXT: s_mov_b32 s3, 0
; GFX12-GISEL-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX12-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_sub_nc_u16 v0.l, v1.l, s2 clamp
-; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX12-GISEL-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
; GFX12-GISEL-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
@@ -2123,9 +2133,10 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_16(ptr
; GFX11-SDAG-NEXT: s_mov_b32 s3, 0
; GFX11-SDAG-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX11-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_sub_nc_u16 v0.l, v1.l, s2 clamp
-; GFX11-SDAG-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX11-SDAG-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
; GFX11-SDAG-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:2048 glc
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
@@ -2151,9 +2162,10 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_16(ptr
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0
; GFX12-SDAG-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX12-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_sub_nc_u16 v0.l, v1.l, s2 clamp
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX12-SDAG-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
; GFX12-SDAG-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
@@ -2364,11 +2376,11 @@ define i8 @global_atomic_usub_sat_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX11-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX11-SDAG-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-SDAG-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_and_b16 v2.h, 0xff, v4.l
-; GFX11-SDAG-NEXT: v_sub_nc_u16 v3.l, v2.h, v2.l clamp
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_sub_nc_u16 v2.h, v2.h, v2.l clamp
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-SDAG-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-SDAG-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -2399,11 +2411,11 @@ define i8 @global_atomic_usub_sat_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX12-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX12-SDAG-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_and_b16 v2.h, 0xff, v4.l
-; GFX12-SDAG-NEXT: v_sub_nc_u16 v3.l, v2.h, v2.l clamp
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_sub_nc_u16 v2.h, v2.h, v2.l clamp
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-SDAG-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
; GFX12-SDAG-NEXT: s_wait_storecnt 0x0
; GFX12-SDAG-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -2614,11 +2626,11 @@ define i8 @global_atomic_usub_sat_offset_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX11-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX11-SDAG-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-SDAG-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_and_b16 v2.h, 0xff, v4.l
-; GFX11-SDAG-NEXT: v_sub_nc_u16 v3.l, v2.h, v2.l clamp
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_sub_nc_u16 v2.h, v2.h, v2.l clamp
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-SDAG-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-SDAG-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:1024 glc
@@ -2649,11 +2661,11 @@ define i8 @global_atomic_usub_sat_offset_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX12-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX12-SDAG-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_and_b16 v2.h, 0xff, v4.l
-; GFX12-SDAG-NEXT: v_sub_nc_u16 v3.l, v2.h, v2.l clamp
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_sub_nc_u16 v2.h, v2.h, v2.l clamp
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-SDAG-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
; GFX12-SDAG-NEXT: s_wait_storecnt 0x0
; GFX12-SDAG-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -2857,9 +2869,10 @@ define void @global_atomic_usub_sat_nortn_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX11-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX11-SDAG-NEXT: v_and_b16 v2.h, 0xff, v4.l
-; GFX11-SDAG-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT: v_sub_nc_u16 v3.l, v2.h, v2.l clamp
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_sub_nc_u16 v2.h, v2.h, v2.l clamp
+; GFX11-SDAG-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-SDAG-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -2890,9 +2903,10 @@ define void @global_atomic_usub_sat_nortn_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX12-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX12-SDAG-NEXT: v_and_b16 v2.h, 0xff, v4.l
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_sub_nc_u16 v3.l, v2.h, v2.l clamp
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_sub_nc_u16 v2.h, v2.h, v2.l clamp
+; GFX12-SDAG-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
; GFX12-SDAG-NEXT: s_wait_storecnt 0x0
; GFX12-SDAG-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -3095,9 +3109,10 @@ define void @global_atomic_usub_sat_offset_nortn_8(ptr addrspace(1) %ptr, i8 %da
; GFX11-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX11-SDAG-NEXT: v_and_b16 v2.h, 0xff, v4.l
-; GFX11-SDAG-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT: v_sub_nc_u16 v3.l, v2.h, v2.l clamp
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_sub_nc_u16 v2.h, v2.h, v2.l clamp
+; GFX11-SDAG-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
; GFX11-SDAG-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-SDAG-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:1024 glc
@@ -3128,9 +3143,10 @@ define void @global_atomic_usub_sat_offset_nortn_8(ptr addrspace(1) %ptr, i8 %da
; GFX12-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX12-SDAG-NEXT: v_and_b16 v2.h, 0xff, v4.l
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-NEXT: v_sub_nc_u16 v3.l, v2.h, v2.l clamp
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_sub_nc_u16 v2.h, v2.h, v2.l clamp
+; GFX12-SDAG-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_and_or_b32 v3, 0xffffff00, v4, v3
; GFX12-SDAG-NEXT: s_wait_storecnt 0x0
; GFX12-SDAG-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -3383,12 +3399,13 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
; GFX11-SDAG-NEXT: s_mov_b32 s2, 0
; GFX11-SDAG-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX11-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, v1
-; GFX11-SDAG-NEXT: v_mov_b16_e32 v1.h, 0
; GFX11-SDAG-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_sub_nc_u16 v1.l, v1.l, s3 clamp
+; GFX11-SDAG-NEXT: v_cvt_u32_u16_e32 v1, v1.l
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_and_or_b32 v1, 0xffffff00, v2, v1
; GFX11-SDAG-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:1024 glc
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
@@ -3419,12 +3436,13 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0
; GFX12-SDAG-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX12-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, v1
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v1.h, 0
; GFX12-SDAG-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_sub_nc_u16 v1.l, v1.l, s2 clamp
+; GFX12-SDAG-NEXT: v_cvt_u32_u16_e32 v1, v1.l
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_and_or_b32 v1, 0xffffff00, v2, v1
; GFX12-SDAG-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
@@ -3650,11 +3668,11 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_8(ptr a
; GFX11-SDAG-NEXT: s_mov_b32 s2, 0
; GFX11-SDAG-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX11-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_and_b16 v0.l, 0xff, v1.l
-; GFX11-SDAG-NEXT: v_mov_b16_e32 v0.h, 0
; GFX11-SDAG-NEXT: v_sub_nc_u16 v0.l, v0.l, s3 clamp
-; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-SDAG-NEXT: v_and_or_b32 v0, 0xffffff00, v1, v0
; GFX11-SDAG-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:1024 glc
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
@@ -3681,11 +3699,11 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_8(ptr a
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0
; GFX12-SDAG-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX12-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_and_b16 v0.l, 0xff, v1.l
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v0.h, 0
; GFX12-SDAG-NEXT: v_sub_nc_u16 v0.l, v0.l, s2 clamp
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX12-SDAG-NEXT: v_and_or_b32 v0, 0xffffff00, v1, v0
; GFX12-SDAG-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:1024 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/atomics-system-scope.ll b/llvm/test/CodeGen/AMDGPU/atomics-system-scope.ll
index e3c375fae7386..56446697371de 100644
--- a/llvm/test/CodeGen/AMDGPU/atomics-system-scope.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomics-system-scope.ll
@@ -396,12 +396,13 @@ define i16 @global_one_as_atomic_min_i16(ptr addrspace(1) %ptr, i16 %val) {
; REAL16-NEXT: ; =>This Inner Loop Header: Depth=1
; REAL16-NEXT: s_wait_loadcnt 0x0
; REAL16-NEXT: v_mov_b32_e32 v7, v5
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; REAL16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; REAL16-NEXT: v_mov_b16_e32 v5.h, 0
-; REAL16-NEXT: v_min_i16 v5.l, v5.l, v2.l
+; REAL16-NEXT: v_min_i16 v2.h, v5.l, v2.l
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; REAL16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; REAL16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; REAL16-NEXT: v_and_or_b32 v6, v7, v4, v5
; REAL16-NEXT: s_wait_xcnt 0x0
; REAL16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -477,12 +478,13 @@ define i16 @global_one_as_atomic_umin_i16(ptr addrspace(1) %ptr, i16 %val) {
; REAL16-NEXT: ; =>This Inner Loop Header: Depth=1
; REAL16-NEXT: s_wait_loadcnt 0x0
; REAL16-NEXT: v_mov_b32_e32 v7, v5
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; REAL16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; REAL16-NEXT: v_mov_b16_e32 v5.h, 0
-; REAL16-NEXT: v_min_u16 v5.l, v5.l, v2.l
+; REAL16-NEXT: v_min_u16 v2.h, v5.l, v2.l
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; REAL16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; REAL16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; REAL16-NEXT: v_and_or_b32 v6, v7, v4, v5
; REAL16-NEXT: s_wait_xcnt 0x0
; REAL16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -558,12 +560,13 @@ define i16 @global_one_as_atomic_max_i16(ptr addrspace(1) %ptr, i16 %val) {
; REAL16-NEXT: ; =>This Inner Loop Header: Depth=1
; REAL16-NEXT: s_wait_loadcnt 0x0
; REAL16-NEXT: v_mov_b32_e32 v7, v5
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; REAL16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; REAL16-NEXT: v_mov_b16_e32 v5.h, 0
-; REAL16-NEXT: v_max_i16 v5.l, v5.l, v2.l
+; REAL16-NEXT: v_max_i16 v2.h, v5.l, v2.l
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; REAL16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; REAL16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; REAL16-NEXT: v_and_or_b32 v6, v7, v4, v5
; REAL16-NEXT: s_wait_xcnt 0x0
; REAL16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -639,12 +642,13 @@ define i16 @global_one_as_atomic_umax_i16(ptr addrspace(1) %ptr, i16 %val) {
; REAL16-NEXT: ; =>This Inner Loop Header: Depth=1
; REAL16-NEXT: s_wait_loadcnt 0x0
; REAL16-NEXT: v_mov_b32_e32 v7, v5
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; REAL16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; REAL16-NEXT: v_mov_b16_e32 v5.h, 0
-; REAL16-NEXT: v_max_u16 v5.l, v5.l, v2.l
+; REAL16-NEXT: v_max_u16 v2.h, v5.l, v2.l
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; REAL16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; REAL16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; REAL16-NEXT: v_and_or_b32 v6, v7, v4, v5
; REAL16-NEXT: s_wait_xcnt 0x0
; REAL16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -1522,12 +1526,13 @@ define i16 @flat_one_as_atomic_min_i16(ptr %ptr, i16 %val) {
; REAL16-NEXT: ; =>This Inner Loop Header: Depth=1
; REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; REAL16-NEXT: v_mov_b32_e32 v7, v5
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; REAL16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; REAL16-NEXT: v_mov_b16_e32 v5.h, 0
-; REAL16-NEXT: v_min_i16 v5.l, v5.l, v2.l
+; REAL16-NEXT: v_min_i16 v2.h, v5.l, v2.l
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; REAL16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; REAL16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; REAL16-NEXT: v_and_or_b32 v6, v7, v4, v5
; REAL16-NEXT: s_wait_xcnt 0x0
; REAL16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[6:7] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -1603,12 +1608,13 @@ define i16 @flat_one_as_atomic_umin_i16(ptr %ptr, i16 %val) {
; REAL16-NEXT: ; =>This Inner Loop Header: Depth=1
; REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; REAL16-NEXT: v_mov_b32_e32 v7, v5
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; REAL16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; REAL16-NEXT: v_mov_b16_e32 v5.h, 0
-; REAL16-NEXT: v_min_u16 v5.l, v5.l, v2.l
+; REAL16-NEXT: v_min_u16 v2.h, v5.l, v2.l
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; REAL16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; REAL16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; REAL16-NEXT: v_and_or_b32 v6, v7, v4, v5
; REAL16-NEXT: s_wait_xcnt 0x0
; REAL16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[6:7] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -1684,12 +1690,13 @@ define i16 @flat_one_as_atomic_max_i16(ptr %ptr, i16 %val) {
; REAL16-NEXT: ; =>This Inner Loop Header: Depth=1
; REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; REAL16-NEXT: v_mov_b32_e32 v7, v5
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; REAL16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; REAL16-NEXT: v_mov_b16_e32 v5.h, 0
-; REAL16-NEXT: v_max_i16 v5.l, v5.l, v2.l
+; REAL16-NEXT: v_max_i16 v2.h, v5.l, v2.l
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; REAL16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; REAL16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; REAL16-NEXT: v_and_or_b32 v6, v7, v4, v5
; REAL16-NEXT: s_wait_xcnt 0x0
; REAL16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[6:7] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -1765,12 +1772,13 @@ define i16 @flat_one_as_atomic_umax_i16(ptr %ptr, i16 %val) {
; REAL16-NEXT: ; =>This Inner Loop Header: Depth=1
; REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; REAL16-NEXT: v_mov_b32_e32 v7, v5
-; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; REAL16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; REAL16-NEXT: v_mov_b16_e32 v5.h, 0
-; REAL16-NEXT: v_max_u16 v5.l, v5.l, v2.l
+; REAL16-NEXT: v_max_u16 v2.h, v5.l, v2.l
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; REAL16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; REAL16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; REAL16-NEXT: v_and_or_b32 v6, v7, v4, v5
; REAL16-NEXT: s_wait_xcnt 0x0
; REAL16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[6:7] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index ef78ce7943af5..dc125ec5840ae 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -37931,10 +37931,9 @@ define bfloat @v_uitofp_i16_to_bf16(i16 %x) #0 {
; GFX11TRUE16-LABEL: v_uitofp_i16_to_bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v1
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -37965,10 +37964,9 @@ define bfloat @v_uitofp_i16_to_bf16(i16 %x) #0 {
; GFX1250TRUE16: ; %bb.0:
; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX1250TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v1
+; GFX1250TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
@@ -41327,11 +41325,12 @@ define amdgpu_ps i32 @s_select_bf16(bfloat inreg %a, bfloat inreg %b, i32 %c) #0
;
; GFX11TRUE16-LABEL: s_select_bf16:
; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, s0
; GFX11TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cndmask_b16 v0.l, s1, v1.l, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_cndmask_b16 v0.l, s1, v0.l, vcc_lo
+; GFX11TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11TRUE16-NEXT: ; return to shader part epilog
;
@@ -41349,11 +41348,12 @@ define amdgpu_ps i32 @s_select_bf16(bfloat inreg %a, bfloat inreg %b, i32 %c) #0
; GFX1250TRUE16-LABEL: s_select_bf16:
; GFX1250TRUE16: ; %bb.0:
; GFX1250TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, s0
; GFX1250TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v0.l, s1, v1.l, vcc_lo
+; GFX1250TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v0.l, s1, v0.l, vcc_lo
+; GFX1250TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1250TRUE16-NEXT: ; return to shader part epilog
;
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index 59bf43637f504..49cd7db570220 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -3344,14 +3344,15 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v1.l, v1.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v0.h, v1.l, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -3468,13 +3469,14 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v1.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v0.h, v1.l, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -3778,14 +3780,15 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v1.l, v1.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v0.h, v1.l, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -3899,13 +3902,14 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v1.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v0.h, v1.l, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -4216,16 +4220,16 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, 0
; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v6.l, v6.l, v5.l
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v5.h, v6.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v6, v5.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
; GFX12-TRUE16-NEXT: .LBB15_4: ; Parent Loop BB15_3 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
@@ -4436,11 +4440,12 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: ; Child Loop BB15_4 Depth 2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v6.l, v6.l, v5.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v5.h, v6.l, v5.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v6, v5.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v6
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index eec2dc25cc61e..4bc0dcdb24855 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -2416,16 +2416,16 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v1.l, v1.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v1.l, v0.h, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -2547,15 +2547,15 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v1.l, v1.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v1.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -2871,16 +2871,16 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v1.l, v1.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v1.l, v0.h, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -2999,15 +2999,15 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v1.l, v1.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v1.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -3330,17 +3330,18 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: ; Child Loop BB12_4 Depth 2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v9, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v4.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v4.h, v4.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v9, v5
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v4.h, v4.h, v4.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v4.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v9, v5
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v11, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v8, v6 :: v_dual_mov_b32 v7, v5
; GFX12-TRUE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
@@ -3557,13 +3558,14 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: ; Child Loop BB12_4 Depth 2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v9, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v4.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v4.h, v4.h, v4.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v4.h, v4.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v4.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v9, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v11, v5
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index 5ed7f52fcb000..0248ddc93ae88 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -2416,16 +2416,16 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v1.l, v1.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v1.l, v0.h, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -2547,15 +2547,15 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v1.l, v1.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v1.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -2871,16 +2871,16 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v1.l, v1.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v1.l, v0.h, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -2999,15 +2999,15 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v1.l, v1.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v1.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -3330,17 +3330,18 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: ; Child Loop BB12_4 Depth 2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v9, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v4.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v5.l, v4.h, v4.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v9, v5
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v4.h, v4.h, v4.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v4.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v9, v5
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v11, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v8, v6 :: v_dual_mov_b32 v7, v5
; GFX12-TRUE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
@@ -3557,13 +3558,14 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: ; Child Loop BB12_4 Depth 2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v9, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v4.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v4.h, v4.h, v4.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v5.l, v4.h, v4.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v4.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v9, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v11, v5
diff --git a/llvm/test/CodeGen/AMDGPU/clamp-modifier.ll b/llvm/test/CodeGen/AMDGPU/clamp-modifier.ll
index 3937b1cb05d3e..222376d7620ce 100644
--- a/llvm/test/CodeGen/AMDGPU/clamp-modifier.ll
+++ b/llvm/test/CodeGen/AMDGPU/clamp-modifier.ll
@@ -1655,9 +1655,9 @@ define amdgpu_kernel void @v_no_clamp_add_src_v2f16_f16_src(ptr addrspace(1) %ou
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_d16_b16 v0, v0, s[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX11-TRUE16-NEXT: v_add_f16_e32 v0.l, 1.0, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: v_pk_max_f16 v0, v0, v0 clamp
; GFX11-TRUE16-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/ctls.ll b/llvm/test/CodeGen/AMDGPU/ctls.ll
index 410b293d7f0e7..e5c70fae90170 100644
--- a/llvm/test/CodeGen/AMDGPU/ctls.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctls.ll
@@ -221,14 +221,14 @@ define i16 @ctls_i16(i16 %x) {
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_ashrrev_i16 v0.h, 15, v0.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_xor_b16 v0.l, v0.l, v0.h
-; GFX11-NEXT: v_mov_b16_e32 v0.h, 0
-; GFX11-NEXT: v_clz_i32_u32_e32 v0, v0
+; GFX11-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_clz_i32_u32_e32 v0, v0
; GFX11-NEXT: v_min_u32_e32 v0, 32, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_add_nc_u32_e32 v0, -16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_add_nc_u16 v0.l, v0.l, -1
; GFX11-NEXT: s_setpc_b64 s[30:31]
%a = ashr i16 %x, 15
diff --git a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
index f072b07761c91..2f2434aa682da 100644
--- a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
@@ -905,9 +905,9 @@ define double @v_uitofp_i8_to_f64(i8 %arg0) nounwind {
; GFX11-TRUE16-LABEL: v_uitofp_i8_to_f64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: v_cvt_f64_u32_e32 v[0:1], v0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
index 779fa0886b368..79acb90e210c7 100644
--- a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
@@ -3159,11 +3159,10 @@ define void @test_dynamic_stackalloc_device_divergent_non_standard_size_i16(i16
; GFX11-SDAG-NEXT: scratch_store_b32 off, v1, s33
; GFX11-SDAG-NEXT: scratch_store_b32 off, v2, s33 offset:4
; GFX11-SDAG-NEXT: s_mov_b32 exec_lo, s0
-; GFX11-SDAG-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-SDAG-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-SDAG-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-SDAG-NEXT: s_add_i32 s32, s32, 16
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, v3, 2, 15
+; GFX11-SDAG-NEXT: v_lshl_add_u32 v0, v0, 2, 15
; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0x7fff0, v0
; GFX11-SDAG-NEXT: s_or_saveexec_b32 s0, -1
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
index 38012e93e9673..cb3e1b4c6e691 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
@@ -8418,12 +8418,13 @@ define half @flat_agent_atomic_fadd_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v5.l, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.h, v5.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8535,12 +8536,13 @@ define half @flat_agent_atomic_fadd_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v5.l, v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.h, v5.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
@@ -8788,12 +8790,13 @@ define half @flat_agent_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v5.l, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.h, v5.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8909,12 +8912,13 @@ define half @flat_agent_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v5.l, v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.h, v5.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
@@ -9169,12 +9173,13 @@ define half @flat_agent_atomic_fadd_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v5.l, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.h, v5.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -9291,12 +9296,13 @@ define half @flat_agent_atomic_fadd_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v5.l, v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.h, v5.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
@@ -9550,11 +9556,11 @@ define void @flat_agent_atomic_fadd_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v3.l, v3.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.h, v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -9663,11 +9669,11 @@ define void @flat_agent_atomic_fadd_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v3.l, v3.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.h, v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
@@ -9908,11 +9914,11 @@ define void @flat_agent_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v3.l, v3.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.h, v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -10025,11 +10031,11 @@ define void @flat_agent_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v3.l, v3.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.h, v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
@@ -10277,11 +10283,11 @@ define void @flat_agent_atomic_fadd_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v3.l, v3.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.h, v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -10395,11 +10401,11 @@ define void @flat_agent_atomic_fadd_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v3.l, v3.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.h, v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
@@ -10637,9 +10643,9 @@ define void @flat_agent_atomic_fadd_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v3.l, v4.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.h, v4.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -10720,9 +10726,9 @@ define void @flat_agent_atomic_fadd_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v3.l, v4.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.h, v4.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
@@ -10910,9 +10916,10 @@ define half @flat_agent_atomic_fadd_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v3.l, v4.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.h, v4.l, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -10997,9 +11004,10 @@ define half @flat_agent_atomic_fadd_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v3.l, v4.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.h, v4.l, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
@@ -11200,12 +11208,13 @@ define half @flat_system_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v5.l, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.h, v5.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -11323,12 +11332,13 @@ define half @flat_system_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v5.l, v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.h, v5.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
@@ -11586,11 +11596,11 @@ define void @flat_system_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v3.l, v3.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.h, v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -11705,11 +11715,11 @@ define void @flat_system_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v3.l, v3.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.h, v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
index ba2cab39dc8e1..a62ea3989480d 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
@@ -6047,14 +6047,14 @@ define half @flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v2.h, v2.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.h, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6171,14 +6171,14 @@ define half @flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v2.h, v2.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.h, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
@@ -6439,14 +6439,14 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6570,14 +6570,14 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
@@ -6845,14 +6845,14 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6977,14 +6977,14 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
@@ -7249,12 +7249,13 @@ define void @flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v2.h, v2.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7369,12 +7370,13 @@ define void @flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v2.h, v2.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.h, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
@@ -7629,12 +7631,13 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7756,12 +7759,13 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
@@ -8023,12 +8027,13 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8151,12 +8156,13 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
@@ -8407,11 +8413,11 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v4.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v2.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.h, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8501,11 +8507,11 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v4.l, v4.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v2.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.h, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
@@ -8708,9 +8714,10 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v4.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v2.h, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.h, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8798,9 +8805,10 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v4.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v2.h, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.h, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
@@ -9011,14 +9019,14 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -9144,14 +9152,14 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
@@ -9422,12 +9430,13 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -9551,12 +9560,13 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
index ea4bca8e32d6e..e98feb4fd0b62 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
@@ -6047,14 +6047,14 @@ define half @flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v5.l, v2.h, v2.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v2.h, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6171,14 +6171,14 @@ define half @flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v5.l, v2.h, v2.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v2.h, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
@@ -6439,14 +6439,14 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v5.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6570,14 +6570,14 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v5.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
@@ -6845,14 +6845,14 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v5.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6977,14 +6977,14 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v5.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
@@ -7249,12 +7249,13 @@ define void @flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v3.l, v2.h, v2.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7369,12 +7370,13 @@ define void @flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v3.l, v2.h, v2.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v2.h, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
@@ -7629,12 +7631,13 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v5.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7756,12 +7759,13 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v5.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
@@ -8023,12 +8027,13 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v5.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8151,12 +8156,13 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v5.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
@@ -8407,11 +8413,11 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v4.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v3.l, v2.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v2.h, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8501,11 +8507,11 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v4.l, v4.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v3.l, v2.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v2.h, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
@@ -8708,9 +8714,10 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v4.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v3.l, v2.h, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v2.h, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8798,9 +8805,10 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v4.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v3.l, v2.h, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v2.h, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
@@ -9011,14 +9019,14 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v5.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -9144,14 +9152,14 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v5.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
@@ -9422,12 +9430,13 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v5.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -9551,12 +9560,13 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v5.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
index 3ccf089b5e3cf..df3e51e8fdc0a 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
@@ -5849,12 +5849,13 @@ define half @flat_agent_atomic_fsub_ret_f16(ptr %ptr, half %val) #0 {
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_sub_f16_e32 v5.l, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_sub_f16_e32 v2.h, v5.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5966,12 +5967,13 @@ define half @flat_agent_atomic_fsub_ret_f16(ptr %ptr, half %val) #0 {
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_sub_f16_e32 v5.l, v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_sub_f16_e32 v2.h, v5.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
@@ -6219,12 +6221,13 @@ define half @flat_agent_atomic_fsub_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_sub_f16_e32 v5.l, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_sub_f16_e32 v2.h, v5.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6340,12 +6343,13 @@ define half @flat_agent_atomic_fsub_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_sub_f16_e32 v5.l, v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_sub_f16_e32 v2.h, v5.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
@@ -6600,12 +6604,13 @@ define half @flat_agent_atomic_fsub_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_sub_f16_e32 v5.l, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_sub_f16_e32 v2.h, v5.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6722,12 +6727,13 @@ define half @flat_agent_atomic_fsub_ret_f16__offset12b_neg(ptr %ptr, half %val)
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_sub_f16_e32 v5.l, v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_sub_f16_e32 v2.h, v5.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
@@ -6981,11 +6987,11 @@ define void @flat_agent_atomic_fsub_noret_f16(ptr %ptr, half %val) #0 {
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_sub_f16_e32 v3.l, v3.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f16_e32 v2.h, v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7094,11 +7100,11 @@ define void @flat_agent_atomic_fsub_noret_f16(ptr %ptr, half %val) #0 {
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_sub_f16_e32 v3.l, v3.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f16_e32 v2.h, v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
@@ -7339,11 +7345,11 @@ define void @flat_agent_atomic_fsub_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_sub_f16_e32 v3.l, v3.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f16_e32 v2.h, v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7456,11 +7462,11 @@ define void @flat_agent_atomic_fsub_noret_f16__offset12b_pos(ptr %ptr, half %val
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_sub_f16_e32 v3.l, v3.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f16_e32 v2.h, v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
@@ -7708,11 +7714,11 @@ define void @flat_agent_atomic_fsub_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_sub_f16_e32 v3.l, v3.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f16_e32 v2.h, v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7826,11 +7832,11 @@ define void @flat_agent_atomic_fsub_noret_f16__offset12b_neg(ptr %ptr, half %val
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_sub_f16_e32 v3.l, v3.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f16_e32 v2.h, v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
@@ -8069,9 +8075,10 @@ define half @flat_agent_atomic_fsub_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_sub_f16_e32 v3.l, v4.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f16_e32 v2.h, v4.l, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8156,9 +8163,10 @@ define half @flat_agent_atomic_fsub_ret_f16__offset12b_pos__align4(ptr %ptr, hal
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_sub_f16_e32 v3.l, v4.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f16_e32 v2.h, v4.l, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
@@ -8349,9 +8357,9 @@ define void @flat_agent_atomic_fsub_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX12-TRUE16-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_sub_f16_e32 v3.l, v4.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f16_e32 v2.h, v4.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8432,9 +8440,9 @@ define void @flat_agent_atomic_fsub_noret_f16__offset12b__align4_pos(ptr %ptr, h
; GFX11-TRUE16-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_sub_f16_e32 v3.l, v4.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f16_e32 v2.h, v4.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
@@ -8631,12 +8639,13 @@ define half @flat_system_atomic_fsub_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_sub_f16_e32 v5.l, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_sub_f16_e32 v2.h, v5.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -8754,12 +8763,13 @@ define half @flat_system_atomic_fsub_ret_f16__offset12b_pos(ptr %ptr, half %val)
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_sub_f16_e32 v5.l, v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_sub_f16_e32 v2.h, v5.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
@@ -9017,11 +9027,11 @@ define void @flat_system_atomic_fsub_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_sub_f16_e32 v3.l, v3.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f16_e32 v2.h, v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -9136,11 +9146,11 @@ define void @flat_system_atomic_fsub_noret_f16__offset12b_pos(ptr %ptr, half %va
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_sub_f16_e32 v3.l, v3.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f16_e32 v2.h, v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
index 35aa5e401afe6..53bd449502e16 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
@@ -2302,7 +2302,7 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi(ptr inreg %
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3]
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX1250-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-TRUE16-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
@@ -2355,7 +2355,7 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128(p
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX1250-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-GISEL-TRUE16-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
@@ -2800,7 +2800,7 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi(ptr inreg %
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3]
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX1250-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1250-GISEL-TRUE16-NEXT: v_and_or_b32 v0, 0xffff, v1, v0
@@ -2857,7 +2857,7 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128(p
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX1250-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX1250-GISEL-TRUE16-NEXT: v_and_or_b32 v0, 0xffff, v1, v0
diff --git a/llvm/test/CodeGen/AMDGPU/fold-int-pow2-with-fmul-or-fdiv.ll b/llvm/test/CodeGen/AMDGPU/fold-int-pow2-with-fmul-or-fdiv.ll
index 401f06925801f..1668a60cf634e 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-int-pow2-with-fmul-or-fdiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/fold-int-pow2-with-fmul-or-fdiv.ll
@@ -631,10 +631,11 @@ define double @fmul_pow_mul_max_pow2(i16 %cnt) nounwind {
; GFX11-TRUE16-LABEL: fmul_pow_mul_max_pow2:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v0.l, 2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: v_cvt_f64_u32_e32 v[0:1], v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mul_f64 v[0:1], 0x40080000, v[0:1]
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1179,12 +1180,13 @@ define double @fmul_pow_shl_cnt_safe(i16 %cnt) nounwind {
; GFX11-TRUE16-LABEL: fmul_pow_shl_cnt_safe:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v0.l, 1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0xff5f3992
; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0x7befffff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: v_cvt_f64_u32_e32 v[0:1], v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mul_f64 v[0:1], v[0:1], s[0:1]
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
index 0d73abf5bf46d..e385261b9bf8a 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
@@ -8847,12 +8847,13 @@ define half @global_agent_atomic_fadd_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX1250-TRUE16-NEXT: v_add_f16_e32 v5.l, v5.l, v2.l
+; GFX1250-TRUE16-NEXT: v_add_f16_e32 v2.h, v5.l, v2.l
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -8937,12 +8938,13 @@ define half @global_agent_atomic_fadd_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v5.l, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.h, v5.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -9054,12 +9056,13 @@ define half @global_agent_atomic_fadd_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v5.l, v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.h, v5.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
@@ -9352,12 +9355,13 @@ define half @global_agent_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX1250-TRUE16-NEXT: v_add_f16_e32 v5.l, v5.l, v2.l
+; GFX1250-TRUE16-NEXT: v_add_f16_e32 v2.h, v5.l, v2.l
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -9443,12 +9447,13 @@ define half @global_agent_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v5.l, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.h, v5.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -9564,12 +9569,13 @@ define half @global_agent_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v5.l, v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.h, v5.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
@@ -9871,12 +9877,13 @@ define half @global_agent_atomic_fadd_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX1250-TRUE16-NEXT: v_add_f16_e32 v5.l, v5.l, v2.l
+; GFX1250-TRUE16-NEXT: v_add_f16_e32 v2.h, v5.l, v2.l
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -9963,12 +9970,13 @@ define half @global_agent_atomic_fadd_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v5.l, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.h, v5.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -10085,12 +10093,13 @@ define half @global_agent_atomic_fadd_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v5.l, v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.h, v5.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
@@ -10391,11 +10400,11 @@ define void @global_agent_atomic_fadd_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_add_f16_e32 v4.l, v4.l, v2.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_add_f16_e32 v2.h, v4.l, v2.l
+; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v2.h
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -10478,11 +10487,11 @@ define void @global_agent_atomic_fadd_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v3.l, v3.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.h, v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -10591,11 +10600,11 @@ define void @global_agent_atomic_fadd_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v3.l, v3.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.h, v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -10880,11 +10889,11 @@ define void @global_agent_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_add_f16_e32 v4.l, v4.l, v2.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_add_f16_e32 v2.h, v4.l, v2.l
+; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v2.h
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -10968,11 +10977,11 @@ define void @global_agent_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v3.l, v3.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.h, v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -11085,11 +11094,11 @@ define void @global_agent_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v3.l, v3.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.h, v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -11383,11 +11392,11 @@ define void @global_agent_atomic_fadd_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_add_f16_e32 v4.l, v4.l, v2.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_add_f16_e32 v2.h, v4.l, v2.l
+; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v2.h
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -11472,11 +11481,11 @@ define void @global_agent_atomic_fadd_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v3.l, v3.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.h, v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -11590,11 +11599,11 @@ define void @global_agent_atomic_fadd_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v3.l, v3.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.h, v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -11879,9 +11888,10 @@ define half @global_agent_atomic_fadd_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v5, v3
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_add_f16_e32 v3.l, v5.l, v2.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_add_f16_e32 v2.h, v5.l, v2.l
+; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v3
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -11947,9 +11957,10 @@ define half @global_agent_atomic_fadd_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v3.l, v4.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.h, v4.l, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -12034,9 +12045,10 @@ define half @global_agent_atomic_fadd_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v3.l, v4.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.h, v4.l, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
@@ -12263,9 +12275,9 @@ define void @global_agent_atomic_fadd_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX1250-TRUE16-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_add_f16_e32 v3.l, v5.l, v2.l
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_add_f16_e32 v2.h, v5.l, v2.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX1250-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v3
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -12328,9 +12340,9 @@ define void @global_agent_atomic_fadd_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX12-TRUE16-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v3.l, v4.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.h, v4.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -12411,9 +12423,9 @@ define void @global_agent_atomic_fadd_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX11-TRUE16-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v3.l, v4.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.h, v4.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
@@ -12642,12 +12654,13 @@ define half @global_system_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v7
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX1250-TRUE16-NEXT: v_add_f16_e32 v5.l, v5.l, v2.l
+; GFX1250-TRUE16-NEXT: v_add_f16_e32 v2.h, v5.l, v2.l
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_SYS
@@ -12733,12 +12746,13 @@ define half @global_system_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v5.l, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.h, v5.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -12856,12 +12870,13 @@ define half @global_system_atomic_fadd_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v5.l, v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.h, v5.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
@@ -13165,11 +13180,11 @@ define void @global_system_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_add_f16_e32 v4.l, v4.l, v2.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_add_f16_e32 v2.h, v4.l, v2.l
+; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v2.h
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_SYS
@@ -13253,11 +13268,11 @@ define void @global_system_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v3.l, v3.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.h, v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -13372,11 +13387,11 @@ define void @global_system_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v3.l, v3.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.h, v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -13674,7 +13689,6 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX1250-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX1250-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v6.h, 0
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -13682,9 +13696,9 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v5.l
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v6
+; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -14262,7 +14276,6 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX1250-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX1250-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v6.h, 0
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -14270,9 +14283,9 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v5.l
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v6
+; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -14871,7 +14884,6 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX1250-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX1250-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v6.h, 0
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -14879,9 +14891,9 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v5.l
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v6
+; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
@@ -15483,13 +15495,12 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v4, v4, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v4, v4, s0
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v3, v7
+; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v4.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -16056,13 +16067,12 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v2, 1.0, v6 op_sel:[0,1,0] op_sel_hi:[1,1,1]
; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v2, s0
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -16650,13 +16660,12 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v2, 1.0, v6 op_sel:[0,1,0] op_sel_hi:[1,1,1]
; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v2, s0
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -17236,13 +17245,12 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v5, v3
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v3, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v3, v3, s0
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX1250-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v4
+; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v3.l
+; GFX1250-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v3
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -17717,13 +17725,12 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX1250-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v3, 1.0, v4 op_sel:[0,1,0] op_sel_hi:[1,1,1]
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v2, s0
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v5
+; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -18189,7 +18196,6 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX1250-TRUE16-NEXT: v_not_b32_e32 v4, v4
; GFX1250-TRUE16-NEXT: .LBB62_1: ; %atomicrmw.start
; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v6.h, 0
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -18197,9 +18203,9 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v5, v5, 1.0, v2 op_sel:[0,1,0] op_sel_hi:[1,1,1]
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v5, v5, s0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v5.l
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v6
+; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX1250-TRUE16-NEXT: v_and_or_b32 v6, v7, v4, v5
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_SYS
@@ -18806,13 +18812,12 @@ define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine
; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v7.h, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v2, 1.0, v6 op_sel:[0,1,0] op_sel_hi:[1,1,1]
; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v2, s0
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v7.l, v2.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v7
+; GFX1250-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
index bf3688338bf8c..1c2b43d85db1b 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
@@ -4470,14 +4470,14 @@ define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v2.h, v2.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.h, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -4594,14 +4594,14 @@ define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v2.h, v2.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.h, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
@@ -4910,14 +4910,14 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5041,14 +5041,14 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
@@ -5366,14 +5366,14 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5498,14 +5498,14 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
@@ -5820,12 +5820,13 @@ define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v2.h, v2.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5940,12 +5941,13 @@ define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v2.h, v2.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.h, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -6248,12 +6250,13 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6375,12 +6378,13 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
@@ -6691,12 +6695,13 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6819,12 +6824,13 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
@@ -7124,11 +7130,11 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v4.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v2.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.h, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7218,11 +7224,11 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v4.l, v4.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v2.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.h, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
@@ -7464,9 +7470,10 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v4.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v2.h, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.h, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7554,9 +7561,10 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v4.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v2.h, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.h, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
@@ -7803,14 +7811,14 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -7936,14 +7944,14 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
@@ -8264,12 +8272,13 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -8393,12 +8402,13 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
index ec43bca792888..8d5413eeda2f0 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
@@ -4470,14 +4470,14 @@ define half @global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v5.l, v2.h, v2.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v2.h, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -4594,14 +4594,14 @@ define half @global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v5.l, v2.h, v2.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v2.h, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
@@ -4910,14 +4910,14 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v5.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5041,14 +5041,14 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v5.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
@@ -5366,14 +5366,14 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v5.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5498,14 +5498,14 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v5.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
@@ -5820,12 +5820,13 @@ define void @global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v3.l, v2.h, v2.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5940,12 +5941,13 @@ define void @global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v3.l, v2.h, v2.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v2.h, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -6248,12 +6250,13 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v5.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6375,12 +6378,13 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v5.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
@@ -6691,12 +6695,13 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v5.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6819,12 +6824,13 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v5.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
@@ -7124,11 +7130,11 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v4.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v3.l, v2.h, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v2.h, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7218,11 +7224,11 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v4.l, v4.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v3.l, v2.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v2.h, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
@@ -7464,9 +7470,10 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v4.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v3.l, v2.h, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v2.h, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7554,9 +7561,10 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v4.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v3.l, v2.h, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v2.h, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
@@ -7803,14 +7811,14 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v5.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -7936,14 +7944,14 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v5.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
@@ -8264,12 +8272,13 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v5.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -8393,12 +8402,13 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v5.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
index f42947b97e4d5..62170ecfc1f92 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
@@ -5221,12 +5221,13 @@ define half @global_agent_atomic_fsub_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_sub_f16_e32 v5.l, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_sub_f16_e32 v2.h, v5.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5338,12 +5339,13 @@ define half @global_agent_atomic_fsub_ret_f16(ptr addrspace(1) %ptr, half %val)
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_sub_f16_e32 v5.l, v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_sub_f16_e32 v2.h, v5.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
@@ -5639,12 +5641,13 @@ define half @global_agent_atomic_fsub_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_sub_f16_e32 v5.l, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_sub_f16_e32 v2.h, v5.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5760,12 +5763,13 @@ define half @global_agent_atomic_fsub_ret_f16__offset12b_pos(ptr addrspace(1) %p
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_sub_f16_e32 v5.l, v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_sub_f16_e32 v2.h, v5.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
@@ -6070,12 +6074,13 @@ define half @global_agent_atomic_fsub_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_sub_f16_e32 v5.l, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_sub_f16_e32 v2.h, v5.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6192,12 +6197,13 @@ define half @global_agent_atomic_fsub_ret_f16__offset12b_neg(ptr addrspace(1) %p
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_sub_f16_e32 v5.l, v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_sub_f16_e32 v2.h, v5.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
@@ -6501,11 +6507,11 @@ define void @global_agent_atomic_fsub_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_sub_f16_e32 v3.l, v3.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f16_e32 v2.h, v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6614,11 +6620,11 @@ define void @global_agent_atomic_fsub_noret_f16(ptr addrspace(1) %ptr, half %val
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_sub_f16_e32 v3.l, v3.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f16_e32 v2.h, v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -6907,11 +6913,11 @@ define void @global_agent_atomic_fsub_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_sub_f16_e32 v3.l, v3.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f16_e32 v2.h, v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7024,11 +7030,11 @@ define void @global_agent_atomic_fsub_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_sub_f16_e32 v3.l, v3.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f16_e32 v2.h, v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -7325,11 +7331,11 @@ define void @global_agent_atomic_fsub_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_sub_f16_e32 v3.l, v3.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f16_e32 v2.h, v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7443,11 +7449,11 @@ define void @global_agent_atomic_fsub_noret_f16__offset12b_neg(ptr addrspace(1)
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_sub_f16_e32 v3.l, v3.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f16_e32 v2.h, v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
@@ -7735,9 +7741,10 @@ define half @global_agent_atomic_fsub_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_sub_f16_e32 v3.l, v4.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f16_e32 v2.h, v4.l, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7822,9 +7829,10 @@ define half @global_agent_atomic_fsub_ret_f16__offset12b_pos__align4(ptr addrspa
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_sub_f16_e32 v3.l, v4.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f16_e32 v2.h, v4.l, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
@@ -8054,9 +8062,9 @@ define void @global_agent_atomic_fsub_noret_f16__offset12b__align4_pos(ptr addrs
; GFX12-TRUE16-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_sub_f16_e32 v3.l, v4.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f16_e32 v2.h, v4.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8137,9 +8145,9 @@ define void @global_agent_atomic_fsub_noret_f16__offset12b__align4_pos(ptr addrs
; GFX11-TRUE16-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_sub_f16_e32 v3.l, v4.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f16_e32 v2.h, v4.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
@@ -8372,12 +8380,13 @@ define half @global_system_atomic_fsub_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX12-TRUE16-NEXT: v_sub_f16_e32 v5.l, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_sub_f16_e32 v2.h, v5.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -8495,12 +8504,13 @@ define half @global_system_atomic_fsub_ret_f16__offset12b_pos(ptr addrspace(1) %
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, 0
-; GFX11-TRUE16-NEXT: v_sub_f16_e32 v5.l, v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_sub_f16_e32 v2.h, v5.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
@@ -8808,11 +8818,11 @@ define void @global_system_atomic_fsub_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_sub_f16_e32 v3.l, v3.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f16_e32 v2.h, v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
@@ -8927,11 +8937,11 @@ define void @global_system_atomic_fsub_noret_f16__offset12b_pos(ptr addrspace(1)
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_sub_f16_e32 v3.l, v3.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f16_e32 v2.h, v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
diff --git a/llvm/test/CodeGen/AMDGPU/idot4u.ll b/llvm/test/CodeGen/AMDGPU/idot4u.ll
index a92fba0c8b24b..199f3c76a17e2 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4u.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4u.ll
@@ -1666,13 +1666,13 @@ define amdgpu_kernel void @notdot4_mixedtypes(ptr addrspace(1) %src1,
; GFX11-DL-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
-; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v0.h, v1.l, v0.l
-; GFX11-DL-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX11-DL-TRUE16-NEXT: v_perm_b32 v1, v5, v5, 0xc0c0302
; GFX11-DL-TRUE16-NEXT: v_mad_u16 v0.l, v2.l, v3.l, v0.l
; GFX11-DL-TRUE16-NEXT: v_perm_b32 v2, v4, v4, 0xc0c0302
-; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-DL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-DL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-DL-TRUE16-NEXT: v_dot4_u32_u8 v0, v2, v1, v0
; GFX11-DL-TRUE16-NEXT: global_store_b16 v6, v0, s[4:5]
; GFX11-DL-TRUE16-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
index 9325595c882f2..bfcde3d5634f2 100644
--- a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
+++ b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
@@ -1929,9 +1929,9 @@ define zeroext i16 @clpeak_umad_pat_i16(i16 zeroext %x, i16 zeroext %y) {
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_mad_u16 v0.h, v0.l, v1.l, v0.l
; GFX11-SDAG-TRUE16-NEXT: v_mad_u16 v0.l, v0.h, v0.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_mad_u16 v0.l, v0.l, v0.h, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: clpeak_umad_pat_i16:
@@ -1959,7 +1959,8 @@ define zeroext i16 @clpeak_umad_pat_i16(i16 zeroext %x, i16 zeroext %y) {
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-FAKE16-LABEL: clpeak_umad_pat_i16:
@@ -1990,9 +1991,9 @@ define zeroext i16 @clpeak_umad_pat_i16(i16 zeroext %x, i16 zeroext %y) {
; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_mad_u16 v0.h, v0.l, v1.l, v0.l
; GFX1200-SDAG-TRUE16-NEXT: v_mad_u16 v0.l, v0.h, v0.l, v0.h
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_mad_u16 v0.l, v0.l, v0.h, v0.l
-; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX1200-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1200-SDAG-FAKE16-LABEL: clpeak_umad_pat_i16:
@@ -2028,7 +2029,8 @@ define zeroext i16 @clpeak_umad_pat_i16(i16 zeroext %x, i16 zeroext %y) {
; GFX1200-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
; GFX1200-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
-; GFX1200-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX1200-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1200-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1200-GISEL-FAKE16-LABEL: clpeak_umad_pat_i16:
@@ -2091,9 +2093,9 @@ define zeroext i16 @clpeak_umad_pat_i16(i16 zeroext %x, i16 zeroext %y) {
; GFX1250-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-SDAG-REAL16-NEXT: v_mad_u16 v0.h, v0.l, v1.l, v0.l
; GFX1250-SDAG-REAL16-NEXT: v_mad_u16 v0.l, v0.h, v0.l, v0.h
-; GFX1250-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-SDAG-REAL16-NEXT: v_mad_u16 v0.l, v0.l, v0.h, v0.l
-; GFX1250-SDAG-REAL16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX1250-SDAG-REAL16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1250-SDAG-REAL16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1250-GISEL-REAL16-LABEL: clpeak_umad_pat_i16:
@@ -2110,7 +2112,8 @@ define zeroext i16 @clpeak_umad_pat_i16(i16 zeroext %x, i16 zeroext %y) {
; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
-; GFX1250-GISEL-REAL16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-REAL16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1250-GISEL-REAL16-NEXT: s_set_pc_i64 s[30:31]
; GFX1250-GISEL-LABEL: clpeak_umad_pat_i16:
; GFX1250-GISEL: ; %bb.0: ; %entry
@@ -10496,9 +10499,9 @@ define zeroext i16 @clpeak_umad_pat_i16_x2(i16 zeroext %x, i16 zeroext %y) {
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_mad_u16 v0.h, v0.l, v0.h, v0.l
; GFX11-SDAG-TRUE16-NEXT: v_mad_u16 v0.l, v0.h, v0.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_mad_u16 v0.l, v0.l, v0.h, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: clpeak_umad_pat_i16_x2:
@@ -10542,7 +10545,8 @@ define zeroext i16 @clpeak_umad_pat_i16_x2(i16 zeroext %x, i16 zeroext %y) {
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
; GFX11-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-FAKE16-LABEL: clpeak_umad_pat_i16_x2:
@@ -10589,9 +10593,9 @@ define zeroext i16 @clpeak_umad_pat_i16_x2(i16 zeroext %x, i16 zeroext %y) {
; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_mad_u16 v0.h, v0.l, v0.h, v0.l
; GFX1200-SDAG-TRUE16-NEXT: v_mad_u16 v0.l, v0.h, v0.l, v0.h
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_mad_u16 v0.l, v0.l, v0.h, v0.l
-; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX1200-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1200-SDAG-FAKE16-LABEL: clpeak_umad_pat_i16_x2:
@@ -10643,7 +10647,8 @@ define zeroext i16 @clpeak_umad_pat_i16_x2(i16 zeroext %x, i16 zeroext %y) {
; GFX1200-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
; GFX1200-GISEL-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
-; GFX1200-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX1200-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1200-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1200-GISEL-FAKE16-LABEL: clpeak_umad_pat_i16_x2:
@@ -10738,9 +10743,9 @@ define zeroext i16 @clpeak_umad_pat_i16_x2(i16 zeroext %x, i16 zeroext %y) {
; GFX1250-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-SDAG-REAL16-NEXT: v_mad_u16 v0.h, v0.l, v0.h, v0.l
; GFX1250-SDAG-REAL16-NEXT: v_mad_u16 v0.l, v0.h, v0.l, v0.h
-; GFX1250-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-SDAG-REAL16-NEXT: v_mad_u16 v0.l, v0.l, v0.h, v0.l
-; GFX1250-SDAG-REAL16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX1250-SDAG-REAL16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1250-SDAG-REAL16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1250-GISEL-REAL16-LABEL: clpeak_umad_pat_i16_x2:
@@ -10767,7 +10772,8 @@ define zeroext i16 @clpeak_umad_pat_i16_x2(i16 zeroext %x, i16 zeroext %y) {
; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
; GFX1250-GISEL-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
-; GFX1250-GISEL-REAL16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-REAL16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1250-GISEL-REAL16-NEXT: s_set_pc_i64 s[30:31]
; GFX1250-GISEL-LABEL: clpeak_umad_pat_i16_x2:
; GFX1250-GISEL: ; %bb.0: ; %entry
@@ -12531,11 +12537,9 @@ define i64 @mul_u24_known_24bit_add64(i16 %x.s, i16 %y.s, i64 %z) {
; GFX11-SDAG-TRUE16-LABEL: mul_u24_known_24bit_add64:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.h, v4.h
+; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v1.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_mad_u64_u32 v[0:1], null, v4, v5, v[2:3]
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -12568,12 +12572,10 @@ define i64 @mul_u24_known_24bit_add64(i16 %x.s, i16 %y.s, i64 %z) {
; GFX1200-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
-; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.h
-; GFX1200-SDAG-TRUE16-NEXT: v_mad_co_u64_u32 v[0:1], null, v4, v0, v[2:3]
+; GFX1200-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX1200-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v1, v[2:3]
; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1200-SDAG-FAKE16-LABEL: mul_u24_known_24bit_add64:
@@ -12633,12 +12635,10 @@ define i64 @mul_u24_known_24bit_add64(i16 %x.s, i16 %y.s, i64 %z) {
; GFX1250-SDAG-REAL16: ; %bb.0:
; GFX1250-SDAG-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-REAL16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX1250-SDAG-REAL16-NEXT: v_mov_b16_e32 v4.l, v0.l
-; GFX1250-SDAG-REAL16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX1250-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-SDAG-REAL16-NEXT: v_mov_b16_e32 v0.h, v4.h
-; GFX1250-SDAG-REAL16-NEXT: v_mad_co_u64_u32 v[0:1], null, v4, v0, v[2:3]
+; GFX1250-SDAG-REAL16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX1250-SDAG-REAL16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
+; GFX1250-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-REAL16-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v1, v[2:3]
; GFX1250-SDAG-REAL16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1250-GISEL-REAL16-LABEL: mul_u24_known_24bit_add64:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
index 71b3b979fa059..947b26df5a569 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
@@ -397,14 +397,13 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1164DAGISEL-TRUE16: ; %bb.0: ; %entry
; GFX1164DAGISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1164DAGISEL-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX1164DAGISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX1164DAGISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
; GFX1164DAGISEL-TRUE16-NEXT: s_mov_b64 s[0:1], exec
; GFX1164DAGISEL-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1164DAGISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1164DAGISEL-TRUE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
; GFX1164DAGISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164DAGISEL-TRUE16-NEXT: v_readlane_b32 s4, v3, s3
+; GFX1164DAGISEL-TRUE16-NEXT: v_readlane_b32 s4, v2, s3
; GFX1164DAGISEL-TRUE16-NEXT: s_bitset0_b64 s[0:1], s3
; GFX1164DAGISEL-TRUE16-NEXT: s_and_b32 s2, s2, s4
; GFX1164DAGISEL-TRUE16-NEXT: s_cmp_lg_u64 s[0:1], 0
@@ -417,14 +416,13 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-TRUE16: ; %bb.0: ; %entry
; GFX1132DAGISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1132DAGISEL-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX1132DAGISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX1132DAGISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
; GFX1132DAGISEL-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX1132DAGISEL-TRUE16-NEXT: s_mov_b32 s0, -1
; GFX1132DAGISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1132DAGISEL-TRUE16-NEXT: s_ctz_i32_b32 s2, s1
; GFX1132DAGISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132DAGISEL-TRUE16-NEXT: v_readlane_b32 s3, v3, s2
+; GFX1132DAGISEL-TRUE16-NEXT: v_readlane_b32 s3, v2, s2
; GFX1132DAGISEL-TRUE16-NEXT: s_bitset0_b32 s1, s2
; GFX1132DAGISEL-TRUE16-NEXT: s_and_b32 s0, s0, s3
; GFX1132DAGISEL-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
index 1546037f5b452..1ae2a4268ff1e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
@@ -397,14 +397,13 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1164DAGISEL-TRUE16: ; %bb.0: ; %entry
; GFX1164DAGISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1164DAGISEL-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX1164DAGISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX1164DAGISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
; GFX1164DAGISEL-TRUE16-NEXT: s_mov_b64 s[0:1], exec
; GFX1164DAGISEL-TRUE16-NEXT: s_mov_b32 s2, 0
; GFX1164DAGISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1164DAGISEL-TRUE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
; GFX1164DAGISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164DAGISEL-TRUE16-NEXT: v_readlane_b32 s4, v3, s3
+; GFX1164DAGISEL-TRUE16-NEXT: v_readlane_b32 s4, v2, s3
; GFX1164DAGISEL-TRUE16-NEXT: s_bitset0_b64 s[0:1], s3
; GFX1164DAGISEL-TRUE16-NEXT: s_or_b32 s2, s2, s4
; GFX1164DAGISEL-TRUE16-NEXT: s_cmp_lg_u64 s[0:1], 0
@@ -417,14 +416,13 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-TRUE16: ; %bb.0: ; %entry
; GFX1132DAGISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1132DAGISEL-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX1132DAGISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX1132DAGISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
; GFX1132DAGISEL-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX1132DAGISEL-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX1132DAGISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1132DAGISEL-TRUE16-NEXT: s_ctz_i32_b32 s2, s1
; GFX1132DAGISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132DAGISEL-TRUE16-NEXT: v_readlane_b32 s3, v3, s2
+; GFX1132DAGISEL-TRUE16-NEXT: v_readlane_b32 s3, v2, s2
; GFX1132DAGISEL-TRUE16-NEXT: s_bitset0_b32 s1, s2
; GFX1132DAGISEL-TRUE16-NEXT: s_or_b32 s0, s0, s3
; GFX1132DAGISEL-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
index b08b8751bfde2..4d5bd7c29417a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
@@ -409,14 +409,13 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1164DAGISEL-TRUE16: ; %bb.0: ; %entry
; GFX1164DAGISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1164DAGISEL-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX1164DAGISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX1164DAGISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
; GFX1164DAGISEL-TRUE16-NEXT: s_mov_b64 s[0:1], exec
; GFX1164DAGISEL-TRUE16-NEXT: s_mov_b32 s2, 0
; GFX1164DAGISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1164DAGISEL-TRUE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
; GFX1164DAGISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164DAGISEL-TRUE16-NEXT: v_readlane_b32 s4, v3, s3
+; GFX1164DAGISEL-TRUE16-NEXT: v_readlane_b32 s4, v2, s3
; GFX1164DAGISEL-TRUE16-NEXT: s_bitset0_b64 s[0:1], s3
; GFX1164DAGISEL-TRUE16-NEXT: s_max_u32 s2, s2, s4
; GFX1164DAGISEL-TRUE16-NEXT: s_cmp_lg_u64 s[0:1], 0
@@ -429,14 +428,13 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-TRUE16: ; %bb.0: ; %entry
; GFX1132DAGISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1132DAGISEL-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX1132DAGISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX1132DAGISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
; GFX1132DAGISEL-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX1132DAGISEL-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX1132DAGISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1132DAGISEL-TRUE16-NEXT: s_ctz_i32_b32 s2, s1
; GFX1132DAGISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132DAGISEL-TRUE16-NEXT: v_readlane_b32 s3, v3, s2
+; GFX1132DAGISEL-TRUE16-NEXT: v_readlane_b32 s3, v2, s2
; GFX1132DAGISEL-TRUE16-NEXT: s_bitset0_b32 s1, s2
; GFX1132DAGISEL-TRUE16-NEXT: s_max_u32 s0, s0, s3
; GFX1132DAGISEL-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
index 002406bb70227..e787784a656ee 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
@@ -397,14 +397,13 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1164DAGISEL-TRUE16: ; %bb.0: ; %entry
; GFX1164DAGISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1164DAGISEL-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX1164DAGISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX1164DAGISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
; GFX1164DAGISEL-TRUE16-NEXT: s_mov_b64 s[0:1], exec
; GFX1164DAGISEL-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX1164DAGISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1164DAGISEL-TRUE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
; GFX1164DAGISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164DAGISEL-TRUE16-NEXT: v_readlane_b32 s4, v3, s3
+; GFX1164DAGISEL-TRUE16-NEXT: v_readlane_b32 s4, v2, s3
; GFX1164DAGISEL-TRUE16-NEXT: s_bitset0_b64 s[0:1], s3
; GFX1164DAGISEL-TRUE16-NEXT: s_min_u32 s2, s2, s4
; GFX1164DAGISEL-TRUE16-NEXT: s_cmp_lg_u64 s[0:1], 0
@@ -417,14 +416,13 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-TRUE16: ; %bb.0: ; %entry
; GFX1132DAGISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1132DAGISEL-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX1132DAGISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX1132DAGISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
; GFX1132DAGISEL-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX1132DAGISEL-TRUE16-NEXT: s_mov_b32 s0, -1
; GFX1132DAGISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1132DAGISEL-TRUE16-NEXT: s_ctz_i32_b32 s2, s1
; GFX1132DAGISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132DAGISEL-TRUE16-NEXT: v_readlane_b32 s3, v3, s2
+; GFX1132DAGISEL-TRUE16-NEXT: v_readlane_b32 s3, v2, s2
; GFX1132DAGISEL-TRUE16-NEXT: s_bitset0_b32 s1, s2
; GFX1132DAGISEL-TRUE16-NEXT: s_min_u32 s0, s0, s3
; GFX1132DAGISEL-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
index a6298a6fdd9c1..84bb41ad62c57 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
@@ -483,14 +483,13 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1164DAGISEL-TRUE16: ; %bb.0: ; %entry
; GFX1164DAGISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1164DAGISEL-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX1164DAGISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX1164DAGISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
; GFX1164DAGISEL-TRUE16-NEXT: s_mov_b64 s[0:1], exec
; GFX1164DAGISEL-TRUE16-NEXT: s_mov_b32 s2, 0
; GFX1164DAGISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1164DAGISEL-TRUE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
; GFX1164DAGISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164DAGISEL-TRUE16-NEXT: v_readlane_b32 s4, v3, s3
+; GFX1164DAGISEL-TRUE16-NEXT: v_readlane_b32 s4, v2, s3
; GFX1164DAGISEL-TRUE16-NEXT: s_bitset0_b64 s[0:1], s3
; GFX1164DAGISEL-TRUE16-NEXT: s_xor_b32 s2, s2, s4
; GFX1164DAGISEL-TRUE16-NEXT: s_cmp_lg_u64 s[0:1], 0
@@ -503,14 +502,13 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-TRUE16: ; %bb.0: ; %entry
; GFX1132DAGISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1132DAGISEL-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX1132DAGISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX1132DAGISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
; GFX1132DAGISEL-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX1132DAGISEL-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX1132DAGISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
; GFX1132DAGISEL-TRUE16-NEXT: s_ctz_i32_b32 s2, s1
; GFX1132DAGISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132DAGISEL-TRUE16-NEXT: v_readlane_b32 s3, v3, s2
+; GFX1132DAGISEL-TRUE16-NEXT: v_readlane_b32 s3, v2, s2
; GFX1132DAGISEL-TRUE16-NEXT: s_bitset0_b32 s1, s2
; GFX1132DAGISEL-TRUE16-NEXT: s_xor_b32 s0, s0, s3
; GFX1132DAGISEL-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
index b0da72184f050..bf596a0f88e03 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
@@ -1248,27 +1248,28 @@ define half @local_atomic_fadd_ret_f16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, -4, v0
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: ds_load_b32 v3, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v0, 0xffff
+; GFX12-TRUE16-NEXT: ds_load_b32 v2, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 24, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v2
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v3
; GFX12-TRUE16-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v0, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v3.l, 4.0, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v3
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v0, v4
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.l, 4.0, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v0, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v4, v3, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v1, v3, v4
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v1, v2, v4
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -1276,7 +1277,7 @@ define half @local_atomic_fadd_ret_f16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v0, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v0, v2
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: local_atomic_fadd_ret_f16:
@@ -1357,34 +1358,35 @@ define half @local_atomic_fadd_ret_f16(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, -4, v0
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: ds_load_b32 v3, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v0, 0xffff
+; GFX11-TRUE16-NEXT: ds_load_b32 v2, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 24, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v2
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v3
; GFX11-TRUE16-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v0, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v3.l, 4.0, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v3
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v0, v4
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.l, 4.0, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v0, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v4, v3, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v1, v3, v4
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v1, v2, v4
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v0, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v0, v2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: local_atomic_fadd_ret_f16:
@@ -1638,12 +1640,13 @@ define half @local_atomic_fadd_ret_f16__offset(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v1, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
; GFX12-TRUE16-NEXT: v_add_f16_e32 v3.l, 4.0, v3.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v3.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v4
@@ -1752,12 +1755,13 @@ define half @local_atomic_fadd_ret_f16__offset(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v1, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
; GFX11-TRUE16-NEXT: v_add_f16_e32 v3.l, 4.0, v3.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v3.l
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v4
@@ -2030,12 +2034,13 @@ define void @local_atomic_fadd_noret_f16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v0, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
; GFX12-TRUE16-NEXT: v_add_f16_e32 v4.l, 4.0, v4.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v4.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v4
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v1, v4, v2
@@ -2136,12 +2141,13 @@ define void @local_atomic_fadd_noret_f16(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v0, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
; GFX11-TRUE16-NEXT: v_add_f16_e32 v4.l, 4.0, v4.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v4.l
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v4
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v1, v4, v2
@@ -2401,11 +2407,11 @@ define void @local_atomic_fadd_noret_f16__offset(ptr addrspace(3) %ptr) nounwind
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v1, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_add_f16_e32 v4.l, 4.0, v4.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v4.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v4
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
@@ -2511,11 +2517,11 @@ define void @local_atomic_fadd_noret_f16__offset(ptr addrspace(3) %ptr) nounwind
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v1, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f16_e32 v4.l, 4.0, v4.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v4.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v4
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
@@ -2775,9 +2781,10 @@ define half @local_atomic_fadd_ret_f16__offset__align4(ptr addrspace(3) %ptr) no
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_add_f16_e32 v1.l, 4.0, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v1
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:65534
@@ -2859,9 +2866,10 @@ define half @local_atomic_fadd_ret_f16__offset__align4(ptr addrspace(3) %ptr) no
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f16_e32 v1.l, 4.0, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v1
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:65534
@@ -3072,8 +3080,8 @@ define void @local_atomic_fadd_noret_f16__offset__align4(ptr addrspace(3) %ptr)
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.l, 4.0, v1.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
@@ -3152,8 +3160,8 @@ define void @local_atomic_fadd_noret_f16__offset__align4(ptr addrspace(3) %ptr)
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.l, 4.0, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
index 43d77d0afa8c6..3729866cc7b07 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
@@ -787,14 +787,14 @@ define half @local_atomic_fmax_ret_f16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v0, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v3.l, v3.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, 4.0, v3.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v3.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v1, v3, v4
@@ -900,14 +900,14 @@ define half @local_atomic_fmax_ret_f16(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v0, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v3.l, v3.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, 4.0, v3.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v1, v3, v4
@@ -1179,14 +1179,14 @@ define half @local_atomic_fmax_ret_f16__offset(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v1, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v3.l, v3.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, 4.0, v3.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v3.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v4
@@ -1297,14 +1297,14 @@ define half @local_atomic_fmax_ret_f16__offset(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v1, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v3.l, v3.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, 4.0, v3.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v4
@@ -1582,14 +1582,14 @@ define void @local_atomic_fmax_noret_f16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v0, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v4.l, v4.l, v4.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v4.l, 4.0, v4.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v4.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v4
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v1, v4, v2
@@ -1692,14 +1692,14 @@ define void @local_atomic_fmax_noret_f16(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v0, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
; GFX11-TRUE16-NEXT: v_max_f16_e32 v4.l, v4.l, v4.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v4.l, 4.0, v4.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v4.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v4
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v1, v4, v2
@@ -1964,12 +1964,13 @@ define void @local_atomic_fmax_noret_f16__offset(ptr addrspace(3) %ptr) nounwind
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v1, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v4.l, v4.l, v4.l
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v4.l, 4.0, v4.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v4.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v4
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
@@ -2078,12 +2079,13 @@ define void @local_atomic_fmax_noret_f16__offset(ptr addrspace(3) %ptr) nounwind
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v1, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v4.l, v4.l, v4.l
; GFX11-TRUE16-NEXT: v_max_f16_e32 v4.l, 4.0, v4.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v4.l
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v4
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
@@ -2349,11 +2351,11 @@ define half @local_atomic_fmax_ret_f16__offset__align4(ptr addrspace(3) %ptr) no
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v1.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v1.l, 4.0, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
; GFX12-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v1
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:65534
@@ -2437,11 +2439,11 @@ define half @local_atomic_fmax_ret_f16__offset__align4(ptr addrspace(3) %ptr) no
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v1.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_max_f16_e32 v1.l, 4.0, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
; GFX11-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v1
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:65534
@@ -2657,9 +2659,10 @@ define void @local_atomic_fmax_noret_f16__offset__align4(ptr addrspace(3) %ptr)
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v1.l, v1.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, 4.0, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
@@ -2741,9 +2744,10 @@ define void @local_atomic_fmax_noret_f16__offset__align4(ptr addrspace(3) %ptr)
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.l, v1.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.l, 4.0, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
index 1759a212c6ea3..edb5a5e239038 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
@@ -787,14 +787,14 @@ define half @local_atomic_fmin_ret_f16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v0, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v3.l, v3.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v3.l, 4.0, v3.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v3.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v1, v3, v4
@@ -900,14 +900,14 @@ define half @local_atomic_fmin_ret_f16(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v0, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v3.l, v3.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_min_f16_e32 v3.l, 4.0, v3.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v1, v3, v4
@@ -1179,14 +1179,14 @@ define half @local_atomic_fmin_ret_f16__offset(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v1, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v3.l, v3.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v3.l, 4.0, v3.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v3.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v4
@@ -1297,14 +1297,14 @@ define half @local_atomic_fmin_ret_f16__offset(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v1, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v3.l, v3.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_min_f16_e32 v3.l, 4.0, v3.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v4
@@ -1582,14 +1582,14 @@ define void @local_atomic_fmin_noret_f16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v0, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v4.l, v4.l, v4.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v4.l, 4.0, v4.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v4.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v4
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v1, v4, v2
@@ -1692,14 +1692,14 @@ define void @local_atomic_fmin_noret_f16(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v0, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
; GFX11-TRUE16-NEXT: v_max_f16_e32 v4.l, v4.l, v4.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_min_f16_e32 v4.l, 4.0, v4.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v4.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v4
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v1, v4, v2
@@ -1964,12 +1964,13 @@ define void @local_atomic_fmin_noret_f16__offset(ptr addrspace(3) %ptr) nounwind
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v1, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v4.l, v4.l, v4.l
; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v4.l, 4.0, v4.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v4.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v4
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
@@ -2078,12 +2079,13 @@ define void @local_atomic_fmin_noret_f16__offset(ptr addrspace(3) %ptr) nounwind
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v1, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v4.l, v4.l, v4.l
; GFX11-TRUE16-NEXT: v_min_f16_e32 v4.l, 4.0, v4.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v4.l
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v4
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
@@ -2349,11 +2351,11 @@ define half @local_atomic_fmin_ret_f16__offset__align4(ptr addrspace(3) %ptr) no
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v1.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v1.l, 4.0, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
; GFX12-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v1
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:65534
@@ -2437,11 +2439,11 @@ define half @local_atomic_fmin_ret_f16__offset__align4(ptr addrspace(3) %ptr) no
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v1.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_min_f16_e32 v1.l, 4.0, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
; GFX11-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v1
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:65534
@@ -2657,9 +2659,10 @@ define void @local_atomic_fmin_noret_f16__offset__align4(ptr addrspace(3) %ptr)
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v1.l, v1.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.l, 4.0, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
@@ -2741,9 +2744,10 @@ define void @local_atomic_fmin_noret_f16__offset__align4(ptr addrspace(3) %ptr)
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.l, v1.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.l, 4.0, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
index 5feb76edf3d17..8508f22d10351 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
@@ -1710,27 +1710,28 @@ define half @local_atomic_fsub_ret_f16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, -4, v0
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: ds_load_b32 v3, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v0, 0xffff
+; GFX12-TRUE16-NEXT: ds_load_b32 v2, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 24, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v2
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v3
; GFX12-TRUE16-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v0, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v3.l, -4.0, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v3
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v0, v4
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.l, -4.0, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v0, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v4, v3, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v1, v3, v4
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v1, v2, v4
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -1738,7 +1739,7 @@ define half @local_atomic_fsub_ret_f16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v0, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v0, v2
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: local_atomic_fsub_ret_f16:
@@ -1819,34 +1820,35 @@ define half @local_atomic_fsub_ret_f16(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, -4, v0
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: ds_load_b32 v3, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v0, 0xffff
+; GFX11-TRUE16-NEXT: ds_load_b32 v2, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 24, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v2
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v3
; GFX11-TRUE16-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v0, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v3.l, -4.0, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v0, v3
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v0, v4
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.l, -4.0, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v0, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v4, v3, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v1, v3, v4
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v1, v2, v4
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v0, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v0, v2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: local_atomic_fsub_ret_f16:
@@ -2100,12 +2102,13 @@ define half @local_atomic_fsub_ret_f16__offset(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v1, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
; GFX12-TRUE16-NEXT: v_add_f16_e32 v3.l, -4.0, v3.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v3.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v4
@@ -2214,12 +2217,13 @@ define half @local_atomic_fsub_ret_f16__offset(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v1, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
; GFX11-TRUE16-NEXT: v_add_f16_e32 v3.l, -4.0, v3.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v3.l
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v1, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v2, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v4
@@ -2492,12 +2496,13 @@ define void @local_atomic_fsub_noret_f16(ptr addrspace(3) %ptr) nounwind {
; GFX12-TRUE16-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v0, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
; GFX12-TRUE16-NEXT: v_add_f16_e32 v4.l, -4.0, v4.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v4.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v4
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v1, v4, v2
@@ -2598,12 +2603,13 @@ define void @local_atomic_fsub_noret_f16(ptr addrspace(3) %ptr) nounwind {
; GFX11-TRUE16-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v0, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
; GFX11-TRUE16-NEXT: v_add_f16_e32 v4.l, -4.0, v4.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v4.l
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v0, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v2, v3, v4
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v1, v4, v2
@@ -2863,11 +2869,11 @@ define void @local_atomic_fsub_noret_f16__offset(ptr addrspace(3) %ptr) nounwind
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v1, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_add_f16_e32 v4.l, -4.0, v4.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v4.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v4
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
@@ -2973,11 +2979,11 @@ define void @local_atomic_fsub_noret_f16__offset(ptr addrspace(3) %ptr) nounwind
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, v1, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f16_e32 v4.l, -4.0, v4.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v4, v4.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, v1, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v4, v3, v2, v4
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
@@ -3237,9 +3243,10 @@ define half @local_atomic_fsub_ret_f16__offset__align4(ptr addrspace(3) %ptr) no
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_add_f16_e32 v1.l, -4.0, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v1
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:65534
@@ -3321,9 +3328,10 @@ define half @local_atomic_fsub_ret_f16__offset__align4(ptr addrspace(3) %ptr) no
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f16_e32 v1.l, -4.0, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v1, 0xffff0000, v2, v1
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v1, v0, v1, v2 offset:65534
@@ -3534,8 +3542,8 @@ define void @local_atomic_fsub_noret_f16__offset__align4(ptr addrspace(3) %ptr)
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: v_add_f16_e32 v2.l, -4.0, v1.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
; GFX12-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
@@ -3614,8 +3622,8 @@ define void @local_atomic_fsub_noret_f16__offset__align4(ptr addrspace(3) %ptr)
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_f16_e32 v2.l, -4.0, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
; GFX11-TRUE16-NEXT: v_and_or_b32 v2, 0xffff0000, v1, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v1 offset:65534
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll
index 1a5b6daf2bdd0..f5c005c11e2b6 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-lo-bf16.ll
@@ -486,7 +486,8 @@ define i32 @mixlo_zext(float %src0, float %src1, float %src2) #0 {
; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
; GFX1250-REAL16-NEXT: v_fma_mixlo_bf16 v0, v0, v1, v2
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-REAL16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
%result = call float @llvm.fmuladd.f32(float %src0, float %src1, float %src2)
%cvt.result = fptrunc float %result to bfloat
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
index d338967abe34f..752101c69eb2c 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
@@ -2500,7 +2500,8 @@ define i32 @mixlo_zext(float %src0, float %src1, float %src2) #0 {
; SDAG-GFX1100-TRUE16: ; %bb.0:
; SDAG-GFX1100-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SDAG-GFX1100-TRUE16-NEXT: v_fma_mixlo_f16 v0, v0, v1, v2
-; SDAG-GFX1100-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; SDAG-GFX1100-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; SDAG-GFX1100-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; SDAG-GFX1100-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-GFX1100-FAKE16-LABEL: mixlo_zext:
diff --git a/llvm/test/CodeGen/AMDGPU/mad.u16.ll b/llvm/test/CodeGen/AMDGPU/mad.u16.ll
index fbf8011fd40c9..cfe14851a1291 100644
--- a/llvm/test/CodeGen/AMDGPU/mad.u16.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad.u16.ll
@@ -179,7 +179,8 @@ define i32 @v_mad_u16_zext(i16 %arg0, i16 %arg1, i16 %arg2) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mad_u16 v0.l, v0.l, v1.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_mad_u16_zext:
@@ -221,9 +222,10 @@ define i64 @v_mad_u16_zext64(i16 %arg0, i16 %arg1, i16 %arg2) {
; GFX11-TRUE16-LABEL: v_mad_u16_zext64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX11-TRUE16-NEXT: v_mad_u16 v0.l, v0.l, v1.l, v2.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_mad_u16_zext64:
diff --git a/llvm/test/CodeGen/AMDGPU/preserve-hi16.ll b/llvm/test/CodeGen/AMDGPU/preserve-hi16.ll
index 93f4ea37117ba..4e343f7e2cd3f 100644
--- a/llvm/test/CodeGen/AMDGPU/preserve-hi16.ll
+++ b/llvm/test/CodeGen/AMDGPU/preserve-hi16.ll
@@ -374,7 +374,7 @@ define i32 @shl_i16_zext_i32(i16 %x, i16 %y) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shl_i16_zext_i32:
@@ -412,7 +412,7 @@ define i32 @lshr_i16_zext_i32(i16 %x, i16 %y) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: lshr_i16_zext_i32:
@@ -450,7 +450,7 @@ define i32 @ashr_i16_zext_i32(i16 %x, i16 %y) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_ashrrev_i16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: ashr_i16_zext_i32:
@@ -488,7 +488,7 @@ define i32 @add_u16_zext_i32(i16 %x, i16 %y) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: add_u16_zext_i32:
@@ -526,7 +526,7 @@ define i32 @sub_u16_zext_i32(i16 %x, i16 %y) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_sub_nc_u16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: sub_u16_zext_i32:
@@ -564,7 +564,7 @@ define i32 @mul_lo_u16_zext_i32(i16 %x, i16 %y) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: mul_lo_u16_zext_i32:
@@ -602,7 +602,7 @@ define i32 @min_u16_zext_i32(i16 %x, i16 %y) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: min_u16_zext_i32:
@@ -641,7 +641,7 @@ define i32 @min_i16_zext_i32(i16 %x, i16 %y) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: min_i16_zext_i32:
@@ -680,7 +680,7 @@ define i32 @max_u16_zext_i32(i16 %x, i16 %y) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: max_u16_zext_i32:
@@ -719,7 +719,7 @@ define i32 @max_i16_zext_i32(i16 %x, i16 %y) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: max_i16_zext_i32:
@@ -758,7 +758,7 @@ define i32 @zext_fadd_f16(half %x, half %y) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: zext_fadd_f16:
@@ -797,10 +797,8 @@ define i32 @zext_fma_f16(half %x, half %y, half %z) {
; GFX11-TRUE16-LABEL: zext_fma_f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: v_fmac_f16_e32 v0.l, v0.h, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX11-TRUE16-NEXT: v_fmac_f16_e32 v2.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: zext_fma_f16:
@@ -840,7 +838,7 @@ define i32 @zext_div_fixup_f16(half %x, half %y, half %z) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_div_fixup_f16 v0.l, v0.l, v1.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: zext_div_fixup_f16:
@@ -882,7 +880,7 @@ define i32 @zext_fptrunc_f16(float %x) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: zext_fptrunc_f16:
@@ -930,7 +928,7 @@ define i32 @zext_fptrunc_fma_f16(float %x, float %y, float %z) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_fma_mixlo_f16 v0, v0, v1, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: zext_fptrunc_fma_f16:
diff --git a/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll b/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
index 74bd338c7c022..5f534f58f7342 100644
--- a/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
+++ b/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
@@ -1158,7 +1158,8 @@ define i32 @v_mul_add_1_i16_zext_result(i16 %x, i16 %y) {
; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
; GFX1250-REAL16-NEXT: v_mad_u16 v0.l, v0.l, v1.l, v0.l
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-REAL16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX13-FAKE16-LABEL: v_mul_add_1_i16_zext_result:
@@ -1181,7 +1182,8 @@ define i32 @v_mul_add_1_i16_zext_result(i16 %x, i16 %y) {
; GFX13-REAL16-NEXT: s_wait_bvhcnt 0x0
; GFX13-REAL16-NEXT: s_wait_kmcnt 0x0
; GFX13-REAL16-NEXT: v_mad_u16 v0.l, v0.l, v1.l, v0.l
-; GFX13-REAL16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX13-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX13-REAL16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX13-REAL16-NEXT: s_set_pc_i64 s[30:31]
%add = add i16 %y, 1
%mul = mul i16 %x, %add
diff --git a/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll b/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
index eb7e9756c0079..bddf0bc019c06 100644
--- a/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
+++ b/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
@@ -1529,8 +1529,9 @@ define amdgpu_kernel void @v_test_i16_x_sub_64_zext_to_i32(ptr addrspace(1) %out
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: global_load_d16_b16 v0, v0, s[2:3]
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX11-SDAG-TRUE16-NEXT: v_sub_nc_u16 v0.l, v0.l, 64
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-SDAG-TRUE16-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11-SDAG-TRUE16-NEXT: s_endpgm
;
@@ -1560,8 +1561,9 @@ define amdgpu_kernel void @v_test_i16_x_sub_64_zext_to_i32(ptr addrspace(1) %out
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-TRUE16-NEXT: global_load_d16_b16 v0, v0, s[2:3]
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
; GFX11-GISEL-TRUE16-NEXT: v_add_nc_u16 v0.l, 0xffc0, v0.l
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-GISEL-TRUE16-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11-GISEL-TRUE16-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/srl-bitcast-bv.ll b/llvm/test/CodeGen/AMDGPU/srl-bitcast-bv.ll
index ee5cee5e86024..b1673787b1d57 100644
--- a/llvm/test/CodeGen/AMDGPU/srl-bitcast-bv.ll
+++ b/llvm/test/CodeGen/AMDGPU/srl-bitcast-bv.ll
@@ -14,8 +14,7 @@ define i64 @srl_bv_v4i16_extract_last(i16 %a, i16 %b, i16 %c, i16 %d) {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b16_e32 v0.h, 0
-; GFX1250-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX1250-NEXT: v_cvt_u32_u16_e32 v0, v3.l
; GFX1250-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%bv0 = insertelement <4 x i16> poison, i16 %a, i32 0
@@ -66,9 +65,8 @@ define i64 @srl_bv_not_last(i16 %a, i16 %b, i16 %c, i16 %d) {
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_mov_b16_e32 v2.h, v3.l
; GFX1250-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX1250-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX1250-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1250-NEXT: v_cvt_u32_u16_e32 v1, v3.l
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1250-NEXT: v_alignbit_b32 v0, v2, v0, 16
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%bv0 = insertelement <4 x i16> poison, i16 %a, i32 0
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
index c52aa7980b55b..b07542e1161a7 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
@@ -337,14 +337,14 @@ define i8 @test_vector_reduce_add_v4i8(<4 x i8> %v) {
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v1.l, v3.l
; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v0.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -393,14 +393,14 @@ define i8 @test_vector_reduce_add_v4i8(<4 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v1.l, v3.l
; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v0.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -573,21 +573,21 @@ define i8 @test_vector_reduce_add_v8i8(<8 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_add_v8i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v2.l, v6.l
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.h, v3.l, v7.l
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v3.l, v7.l
; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.l, v1.l, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.h, v2.l, v6.l
; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.l, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -649,21 +649,21 @@ define i8 @test_vector_reduce_add_v8i8(<8 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v2.l, v6.l
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.h, v3.l, v7.l
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v3.l, v7.l
; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.l, v1.l, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.h, v2.l, v6.l
; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.l, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -918,25 +918,27 @@ define i8 @test_vector_reduce_add_v16i8(<16 x i8> %v) {
; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.h, v5.l, v13.l
; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.l, v1.l, v9.l
; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v5.l, v7.l, v15.l
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v6.l, v14.l
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v2.l, v2.l, v10.l
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v2.h, v3.l, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v3.l, v4.l, v12.l
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v3.l, v11.l
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v2.h, v4.l, v12.l
; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v8.l
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v3.l, v6.l, v14.l
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v2.l, v2.l, v10.l
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v0.h, v5.l
; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.l, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v2.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v2.l, v2.h, v5.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.l, v1.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v2.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.h, v2.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1030,25 +1032,27 @@ define i8 @test_vector_reduce_add_v16i8(<16 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.h, v5.l, v13.l
; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.l, v1.l, v9.l
; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v5.l, v7.l, v15.l
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v6.l, v14.l
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v2.l, v2.l, v10.l
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v2.h, v3.l, v11.l
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v3.l, v4.l, v12.l
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v3.l, v11.l
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v2.h, v4.l, v12.l
; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v8.l
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v3.l, v6.l, v14.l
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v2.l, v2.l, v10.l
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v0.h, v5.l
; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.l, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v2.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v2.l, v2.h, v5.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.l, v1.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v2.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.h, v2.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll
index 6376d3a462c12..e53a44f82ff14 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll
@@ -416,12 +416,13 @@ define i8 @test_vector_reduce_umax_v4i8(<4 x i8> %v) {
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v1.l, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v0.h, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -480,12 +481,13 @@ define i8 @test_vector_reduce_umax_v4i8(<4 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v1.l, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v0.h, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -731,18 +733,19 @@ define i8 @test_vector_reduce_umax_v8i8(<8 x i8> %v) {
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v0.h, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.l
+; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v1.l, v1.l, v1.h
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.h, v0.h, v3.l, v3.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.h, v1.l, v3.l, v3.h
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v0.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v2.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -819,18 +822,19 @@ define i8 @test_vector_reduce_umax_v8i8(<8 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v0.h, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.l
+; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v1.l, v1.l, v1.h
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.h, v0.h, v3.l, v3.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.h, v1.l, v3.l, v3.h
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v0.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v2.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1206,11 +1210,11 @@ define i8 @test_vector_reduce_umax_v16i8(<16 x i8> %v) {
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v1.h, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v1.l, v0.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1339,11 +1343,11 @@ define i8 @test_vector_reduce_umax_v16i8(<16 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v1.h, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v1.l, v0.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
index 1612fef4638dd..9c6194ab6cace 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
@@ -371,12 +371,13 @@ define i8 @test_vector_reduce_umin_v4i8(<4 x i8> %v) {
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v1.l, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v0.h, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -422,12 +423,13 @@ define i8 @test_vector_reduce_umin_v4i8(<4 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v1.l, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v0.h, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -617,18 +619,19 @@ define i8 @test_vector_reduce_umin_v8i8(<8 x i8> %v) {
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v0.h, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.l
+; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v1.l, v1.l, v1.h
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.h, v0.h, v3.l, v3.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.h, v1.l, v3.l, v3.h
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v0.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v2.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -691,18 +694,19 @@ define i8 @test_vector_reduce_umin_v8i8(<8 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v0.h, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.l
+; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v1.l, v1.l, v1.h
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.h, v0.h, v3.l, v3.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.h, v1.l, v3.l, v3.h
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v0.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v2.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1021,11 +1025,11 @@ define i8 @test_vector_reduce_umin_v16i8(<16 x i8> %v) {
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v1.h, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v1.l, v0.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1142,11 +1146,11 @@ define i8 @test_vector_reduce_umin_v16i8(<16 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v1.h, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v1.l, v0.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
More information about the llvm-commits
mailing list