[llvm] [AMDGPU] Do not copy PHI operands whose class differs only in AV-ness (PR #226213)
Pablo Reble via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 25 13:46:18 PDT 2026
https://github.com/reble updated https://github.com/llvm/llvm-project/pull/226213
>From 0763f9f5fa6187b210baff46c687fb696ca3fbc4 Mon Sep 17 00:00:00 2001
From: Pablo Reble <pablo.reble at amd.com>
Date: Thu, 24 Sep 2026 10:35:48 -0500
Subject: [PATCH 1/3] [AMDGPU] Do not copy PHI operands whose class differs
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 11 +
.../test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll | 773 +++++++++---------
.../CodeGen/AMDGPU/a-v-global-atomicrmw.ll | 504 ++++++------
llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll | 44 +-
.../AMDGPU/av-split-dead-valno-crash.ll | 43 +-
.../CodeGen/AMDGPU/copy-to-reg-frameindex.ll | 3 +-
.../test/CodeGen/AMDGPU/flat-saddr-atomics.ll | 52 +-
.../CodeGen/AMDGPU/insert-waitcnts-merge.ll | 2 +-
.../CodeGen/AMDGPU/local-atomicrmw-fadd.ll | 32 +-
.../CodeGen/AMDGPU/local-atomicrmw-fmax.ll | 64 +-
.../CodeGen/AMDGPU/local-atomicrmw-fmin.ll | 64 +-
.../CodeGen/AMDGPU/local-atomicrmw-fsub.ll | 64 +-
llvm/test/CodeGen/AMDGPU/phi-av-pressure.ll | 2 +-
.../rewrite-mfma-form-spill-cost-reset.ll | 218 ++---
.../si-fix-sgpr-copies-phi-av-classes.mir | 386 +++++++++
15 files changed, 1327 insertions(+), 935 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/si-fix-sgpr-copies-phi-av-classes.mir
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 16ee5f56986804..de141a67e5c5e8 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7765,6 +7765,17 @@ SIInstrInfo::legalizeOperands(MachineInstr &MI,
if (!Op.isReg() || !Op.getReg().isVirtual())
continue;
+ // No copy is needed if the classes differ only in AV-ness, e.g. an av_32
+ // incoming value for a vgpr_32 PHI.
+ const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg());
+ if (!Op.getSubReg() && VRC != &AMDGPU::VReg_1RegClass &&
+ OpRC != &AMDGPU::VReg_1RegClass &&
+ (RI.isVectorSuperClass(OpRC) || RI.isVectorSuperClass(VRC))) {
+ const TargetRegisterClass *Common = RI.getCommonSubClass(OpRC, VRC);
+ if (Common && RI.isVGPRClass(Common))
+ continue;
+ }
+
// MI is a PHI instruction.
MachineBasicBlock *InsertBB = MI.getOperand(I + 1).getMBB();
MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator();
diff --git a/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll b/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll
index cb59bb925aab0b..0d32dd3fe5d268 100644
--- a/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll
+++ b/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll
@@ -11812,27 +11812,27 @@ define void @flat_atomic_fmaximum_v2f16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
+; GFX90A-NEXT: s_mov_b32 s6, 0x5040100
; GFX90A-NEXT: .LBB147_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v2, v3, v4
+; GFX90A-NEXT: v_cmp_o_f16_e32 vcc, v3, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v2, vcc
; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v3, v4 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v3, v4
-; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v2, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_sdwa v2, v5, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s8
+; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s6
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB147_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -11950,27 +11950,27 @@ define void @flat_atomic_fminimum_v2f16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
+; GFX90A-NEXT: s_mov_b32 s6, 0x5040100
; GFX90A-NEXT: .LBB149_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_min_f16 v2, v3, v4
+; GFX90A-NEXT: v_cmp_o_f16_e32 vcc, v3, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v2, vcc
; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v3, v4 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v3, v4
-; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v2, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_sdwa v2, v5, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s8
+; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s6
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB149_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -12092,38 +12092,38 @@ define void @flat_atomic_fadd_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB151_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s7
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB151_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -12225,38 +12225,38 @@ define void @flat_atomic_fsub_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB153_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s7
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB153_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -12397,38 +12397,38 @@ define void @flat_atomic_fmax_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB155_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s7
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB155_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -12569,38 +12569,38 @@ define void @flat_atomic_fmin_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB157_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s7
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB157_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -12741,43 +12741,43 @@ define void @flat_atomic_fmaximum_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB159_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_max_f32_e32 v7, v2, v4
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v2, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_max_f32_e32 v8, v2, v4
-; GFX90A-NEXT: v_max_f32_e32 v9, v7, v6
+; GFX90A-NEXT: v_max_f32_e32 v8, v7, v6
; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v4
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
-; GFX90A-NEXT: v_bfe_u32 v8, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX90A-NEXT: v_add3_u32 v8, v8, v2, s8
-; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v8, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v7, 16, 1
+; GFX90A-NEXT: v_add3_u32 v8, v8, v7, s6
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v8, v9, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s7
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB159_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -12923,43 +12923,43 @@ define void @flat_atomic_fminimum_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB161_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_min_f32_e32 v7, v2, v4
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v2, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_min_f32_e32 v8, v2, v4
-; GFX90A-NEXT: v_min_f32_e32 v9, v7, v6
+; GFX90A-NEXT: v_min_f32_e32 v8, v7, v6
; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v4
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
-; GFX90A-NEXT: v_bfe_u32 v8, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX90A-NEXT: v_add3_u32 v8, v8, v2, s8
-; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v8, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v7, 16, 1
+; GFX90A-NEXT: v_add3_u32 v8, v8, v7, s6
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v8, v9, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s7
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB161_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -13490,28 +13490,27 @@ define void @flat_atomic_nand_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB172_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_and_b32_e32 v0, v1, v4
-; GFX90A-NEXT: v_not_b32_e32 v0, v0
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_and_b32_e32 v2, v3, v4
+; GFX90A-NEXT: v_not_b32_e32 v2, v2
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB172_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; use v0
+; GFX90A-NEXT: ; use v2
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -13519,9 +13518,8 @@ define void @flat_atomic_nand_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
@@ -13529,18 +13527,18 @@ define void @flat_atomic_nand_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: .LBB172_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_bitop3_b32 v0, v1, v4, v1 bitop3:0x3f
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_bitop3_b32 v2, v3, v4, v3 bitop3:0x3f
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB172_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v0
+; GFX950-NEXT: ; use v2
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 10
@@ -14259,29 +14257,28 @@ define void @flat_atomic_usub_cond_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB190_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_sub_u32_e32 v0, v1, v4
-; GFX90A-NEXT: v_cmp_ge_u32_e32 vcc, v1, v4
-; GFX90A-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_sub_u32_e32 v2, v3, v4
+; GFX90A-NEXT: v_cmp_ge_u32_e32 vcc, v3, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB190_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; use v0
+; GFX90A-NEXT: ; use v2
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -14289,30 +14286,29 @@ define void @flat_atomic_usub_cond_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: .LBB190_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_sub_u32_e32 v0, v1, v4
-; GFX950-NEXT: v_cmp_ge_u32_e32 vcc, v1, v4
+; GFX950-NEXT: v_sub_u32_e32 v2, v3, v4
+; GFX950-NEXT: v_cmp_ge_u32_e32 vcc, v3, v4
; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB190_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v0
+; GFX950-NEXT: ; use v2
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 10
@@ -14395,27 +14391,26 @@ define void @flat_atomic_usub_sat_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB192_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_sub_u32_e64 v0, v1, v4 clamp
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_sub_u32_e64 v2, v3, v4 clamp
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB192_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; use v0
+; GFX90A-NEXT: ; use v2
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -14423,9 +14418,8 @@ define void @flat_atomic_usub_sat_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
@@ -14433,18 +14427,18 @@ define void @flat_atomic_usub_sat_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: .LBB192_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_sub_u32_e64 v0, v1, v4 clamp
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_sub_u32_e64 v2, v3, v4 clamp
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB192_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v0
+; GFX950-NEXT: ; use v2
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 10
@@ -17140,18 +17134,18 @@ define void @flat_atomic_udec_wrap_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v[2:3]
+; GFX950-NEXT: ; def v[0:1]
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_cbranch_scc0 .LBB218_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] sc0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-NEXT: flat_atomic_dec_x2 v[2:3], v[2:3], v[0:1] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: s_branch .LBB218_3
; GFX950-NEXT: .LBB218_2:
; GFX950-NEXT: s_mov_b64 s[2:3], -1
-; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-NEXT: .LBB218_3: ; %Flow
; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: s_cselect_b32 s2, 1, 0
@@ -17160,21 +17154,21 @@ define void @flat_atomic_udec_wrap_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s2, s0, -1
-; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s2
+; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s2
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_subrev_co_u32_e32 v4, vcc, 1, v0
-; GFX950-NEXT: v_cmp_gt_u64_e64 s[0:1], v[0:1], v[2:3]
+; GFX950-NEXT: v_subrev_co_u32_e32 v4, vcc, 1, v2
+; GFX950-NEXT: v_cmp_gt_u64_e64 s[0:1], v[2:3], v[0:1]
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_subbrev_co_u32_e32 v5, vcc, 0, v1, vcc
+; GFX950-NEXT: v_subbrev_co_u32_e32 v5, vcc, 0, v3, vcc
; GFX950-NEXT: s_or_b64 vcc, vcc, s[0:1]
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX950-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s2
+; GFX950-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX950-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s2
; GFX950-NEXT: .LBB218_5: ; %atomicrmw.phi
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v[0:1]
+; GFX950-NEXT: ; use v[2:3]
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 10
@@ -17965,27 +17959,26 @@ define void @flat_atomic_fsub_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB226_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_sub_f32_e32 v0, v1, v4
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_sub_f32_e32 v2, v3, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB226_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; use v0
+; GFX90A-NEXT: ; use v2
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -17993,27 +17986,26 @@ define void @flat_atomic_fsub_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: .LBB226_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_sub_f32_e32 v0, v1, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_sub_f32_e32 v2, v3, v4
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB226_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v0
+; GFX950-NEXT: ; use v2
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 10
@@ -18406,9 +18398,8 @@ define void @flat_atomic_fmaximum_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
@@ -18416,18 +18407,18 @@ define void @flat_atomic_fmaximum_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: .LBB232_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_maximum3_f32 v0, v1, v4, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_maximum3_f32 v2, v3, v4, v4
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB232_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v0
+; GFX950-NEXT: ; use v2
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 10
@@ -18544,9 +18535,8 @@ define void @flat_atomic_fminimum_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
@@ -18554,18 +18544,18 @@ define void @flat_atomic_fminimum_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: .LBB234_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_minimum3_f32 v0, v1, v4, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_minimum3_f32 v2, v3, v4, v4
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB234_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v0
+; GFX950-NEXT: ; use v2
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 10
@@ -19264,18 +19254,18 @@ define void @flat_atomic_fmax_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v[2:3]
+; GFX950-NEXT: ; def v[0:1]
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_cbranch_scc0 .LBB240_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] sc0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-NEXT: flat_atomic_max_f64 v[2:3], v[2:3], v[0:1] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: s_branch .LBB240_3
; GFX950-NEXT: .LBB240_2:
; GFX950-NEXT: s_mov_b64 s[2:3], -1
-; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-NEXT: .LBB240_3: ; %Flow
; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: s_cselect_b32 s2, 1, 0
@@ -19284,16 +19274,16 @@ define void @flat_atomic_fmax_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
-; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s0
-; GFX950-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s0
+; GFX950-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
-; GFX950-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]
-; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s0
+; GFX950-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX950-NEXT: v_max_f64 v[0:1], v[4:5], v[0:1]
+; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s0
; GFX950-NEXT: .LBB240_5: ; %atomicrmw.end
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v[0:1]
+; GFX950-NEXT: ; use v[2:3]
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 10
@@ -19470,18 +19460,18 @@ define void @flat_atomic_fmin_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v[2:3]
+; GFX950-NEXT: ; def v[0:1]
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_cbranch_scc0 .LBB242_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] sc0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-NEXT: flat_atomic_min_f64 v[2:3], v[2:3], v[0:1] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: s_branch .LBB242_3
; GFX950-NEXT: .LBB242_2:
; GFX950-NEXT: s_mov_b64 s[2:3], -1
-; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-NEXT: .LBB242_3: ; %Flow
; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: s_cselect_b32 s2, 1, 0
@@ -19490,16 +19480,16 @@ define void @flat_atomic_fmin_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
-; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s0
-; GFX950-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s0
+; GFX950-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
-; GFX950-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]
-; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s0
+; GFX950-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX950-NEXT: v_min_f64 v[0:1], v[4:5], v[0:1]
+; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s0
; GFX950-NEXT: .LBB242_5: ; %atomicrmw.end
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v[0:1]
+; GFX950-NEXT: ; use v[2:3]
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 10
@@ -20095,27 +20085,26 @@ define void @flat_atomic_fadd_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB248_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_add_f16 v0, v1, v4
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_pk_add_f16 v2, v3, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB248_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; use v0
+; GFX90A-NEXT: ; use v2
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -20213,27 +20202,26 @@ define void @flat_atomic_fsub_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB250_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_add_f16 v0, v1, v4 neg_lo:[0,1] neg_hi:[0,1]
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_pk_add_f16 v2, v3, v4 neg_lo:[0,1] neg_hi:[0,1]
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB250_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; use v0
+; GFX90A-NEXT: ; use v2
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -20241,9 +20229,8 @@ define void @flat_atomic_fsub_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
@@ -20251,18 +20238,18 @@ define void @flat_atomic_fsub_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: .LBB250_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_add_f16 v0, v1, v4 neg_lo:[0,1] neg_hi:[0,1]
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_pk_add_f16 v2, v3, v4 neg_lo:[0,1] neg_hi:[0,1]
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB250_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v0
+; GFX950-NEXT: ; use v2
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 10
@@ -20566,28 +20553,28 @@ define void @flat_atomic_fmaximum_v2f16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
+; GFX90A-NEXT: s_mov_b32 s6, 0x5040100
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB255_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v0, v1, v4
+; GFX90A-NEXT: v_cmp_o_f16_e32 vcc, v1, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v0, vcc
; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v1, v4 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v1, v4
-; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v0, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_sdwa v0, v5, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_perm_b32 v0, v0, v6, s8
+; GFX90A-NEXT: v_perm_b32 v0, v0, v6, s6
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB255_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -20671,9 +20658,8 @@ define void @flat_atomic_fmaximum_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
@@ -20681,18 +20667,18 @@ define void @flat_atomic_fmaximum_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: .LBB256_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_maximum3_f16 v0, v1, v4, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_pk_maximum3_f16 v2, v3, v4, v4
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB256_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v0
+; GFX950-NEXT: ; use v2
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 10
@@ -20712,28 +20698,28 @@ define void @flat_atomic_fminimum_v2f16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
+; GFX90A-NEXT: s_mov_b32 s6, 0x5040100
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB257_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_min_f16 v0, v1, v4
+; GFX90A-NEXT: v_cmp_o_f16_e32 vcc, v1, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v0, vcc
; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v1, v4 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v1, v4
-; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v0, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_sdwa v0, v5, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_perm_b32 v0, v0, v6, s8
+; GFX90A-NEXT: v_perm_b32 v0, v0, v6, s6
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB257_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -20817,9 +20803,8 @@ define void @flat_atomic_fminimum_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
@@ -20827,18 +20812,18 @@ define void @flat_atomic_fminimum_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: .LBB258_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_minimum3_f16 v0, v1, v4, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_pk_minimum3_f16 v2, v3, v4, v4
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB258_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v0
+; GFX950-NEXT: ; use v2
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 10
@@ -20862,39 +20847,39 @@ define void @flat_atomic_fadd_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB259_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
; GFX90A-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s7
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB259_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -21003,39 +20988,39 @@ define void @flat_atomic_fsub_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB261_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
; GFX90A-NEXT: v_sub_f32_e32 v0, v0, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s7
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB261_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -21183,39 +21168,39 @@ define void @flat_atomic_fmax_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB263_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
; GFX90A-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s7
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB263_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -21363,39 +21348,39 @@ define void @flat_atomic_fmin_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB265_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
; GFX90A-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s7
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB265_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -21543,44 +21528,44 @@ define void @flat_atomic_fmaximum_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB267_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_max_f32_e32 v7, v0, v4
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v0, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v7, vcc
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc
; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_max_f32_e32 v8, v0, v4
-; GFX90A-NEXT: v_max_f32_e32 v9, v7, v6
+; GFX90A-NEXT: v_max_f32_e32 v8, v7, v6
; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v0, v4
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
-; GFX90A-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX90A-NEXT: v_add3_u32 v8, v8, v0, s8
-; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v8, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v7, 16, 1
+; GFX90A-NEXT: v_add3_u32 v8, v8, v7, s6
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v8, v9, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v7, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v7, v0, s7
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB267_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -21733,44 +21718,44 @@ define void @flat_atomic_fminimum_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB269_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_min_f32_e32 v7, v0, v4
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v0, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v7, vcc
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc
; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_min_f32_e32 v8, v0, v4
-; GFX90A-NEXT: v_min_f32_e32 v9, v7, v6
+; GFX90A-NEXT: v_min_f32_e32 v8, v7, v6
; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v0, v4
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
-; GFX90A-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX90A-NEXT: v_add3_u32 v8, v8, v0, s8
-; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v8, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v7, 16, 1
+; GFX90A-NEXT: v_add3_u32 v8, v8, v7, s6
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v8, v9, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v7, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v7, v0, s7
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB269_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
diff --git a/llvm/test/CodeGen/AMDGPU/a-v-global-atomicrmw.ll b/llvm/test/CodeGen/AMDGPU/a-v-global-atomicrmw.ll
index 4983695ac24ee9..304a8501b904d9 100644
--- a/llvm/test/CodeGen/AMDGPU/a-v-global-atomicrmw.ll
+++ b/llvm/test/CodeGen/AMDGPU/a-v-global-atomicrmw.ll
@@ -7884,27 +7884,27 @@ define void @global_atomic_fmaximum_v2f16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
+; GFX90A-NEXT: s_mov_b32 s6, 0x5040100
; GFX90A-NEXT: .LBB147_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v2, v3, v4
+; GFX90A-NEXT: v_cmp_o_f16_e32 vcc, v3, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v2, vcc
; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v3, v4 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v3, v4
-; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v2, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_sdwa v2, v5, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s8
+; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s6
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB147_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -8022,27 +8022,27 @@ define void @global_atomic_fminimum_v2f16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
+; GFX90A-NEXT: s_mov_b32 s6, 0x5040100
; GFX90A-NEXT: .LBB149_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_min_f16 v2, v3, v4
+; GFX90A-NEXT: v_cmp_o_f16_e32 vcc, v3, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v2, vcc
; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v3, v4 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v3, v4
-; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v2, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_sdwa v2, v5, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s8
+; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s6
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB149_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -8164,38 +8164,38 @@ define void @global_atomic_fadd_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB151_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s7
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB151_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -8297,38 +8297,38 @@ define void @global_atomic_fsub_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB153_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s7
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB153_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -8469,38 +8469,38 @@ define void @global_atomic_fmax_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB155_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s7
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB155_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -8641,38 +8641,38 @@ define void @global_atomic_fmin_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB157_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s7
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB157_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -8813,43 +8813,43 @@ define void @global_atomic_fmaximum_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB159_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_max_f32_e32 v7, v2, v4
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v2, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_max_f32_e32 v8, v2, v4
-; GFX90A-NEXT: v_max_f32_e32 v9, v7, v6
+; GFX90A-NEXT: v_max_f32_e32 v8, v7, v6
; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v4
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
-; GFX90A-NEXT: v_bfe_u32 v8, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX90A-NEXT: v_add3_u32 v8, v8, v2, s8
-; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v8, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v7, 16, 1
+; GFX90A-NEXT: v_add3_u32 v8, v8, v7, s6
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v8, v9, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s7
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB159_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -8995,43 +8995,43 @@ define void @global_atomic_fminimum_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB161_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_min_f32_e32 v7, v2, v4
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v2, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_min_f32_e32 v8, v2, v4
-; GFX90A-NEXT: v_min_f32_e32 v9, v7, v6
+; GFX90A-NEXT: v_min_f32_e32 v8, v7, v6
; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v4
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
-; GFX90A-NEXT: v_bfe_u32 v8, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX90A-NEXT: v_add3_u32 v8, v8, v2, s8
-; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v8, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v7, 16, 1
+; GFX90A-NEXT: v_add3_u32 v8, v8, v7, s6
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v8, v9, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s7
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB161_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -13763,27 +13763,27 @@ define void @global_atomic_fmaximum_v2f16_saddr_ret_a_a(ptr addrspace(1) inreg %
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v3, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_mov_b32_e32 v4, 0x7e00
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
+; GFX90A-NEXT: s_mov_b32 s6, 0x5040100
; GFX90A-NEXT: .LBB255_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v0, v1, v3
+; GFX90A-NEXT: v_cmp_o_f16_e32 vcc, v1, v3
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v4, v0, vcc
; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v1, v3 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v1, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v5, v4, v0, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_sdwa v0, v4, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_perm_b32 v0, v0, v5, s8
+; GFX90A-NEXT: v_perm_b32 v0, v0, v5, s6
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB255_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -13905,27 +13905,27 @@ define void @global_atomic_fminimum_v2f16_saddr_ret_a_a(ptr addrspace(1) inreg %
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v3, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_mov_b32_e32 v4, 0x7e00
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
+; GFX90A-NEXT: s_mov_b32 s6, 0x5040100
; GFX90A-NEXT: .LBB257_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_min_f16 v0, v1, v3
+; GFX90A-NEXT: v_cmp_o_f16_e32 vcc, v1, v3
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v4, v0, vcc
; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v1, v3 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v1, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v5, v4, v0, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_sdwa v0, v4, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_perm_b32 v0, v0, v5, s8
+; GFX90A-NEXT: v_perm_b32 v0, v0, v5, s6
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB257_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -14051,38 +14051,38 @@ define void @global_atomic_fadd_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg %ptr
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB259_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX90A-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s6
+; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX90A-NEXT: v_add_f32_e32 v5, v5, v4
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v5
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s7
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB259_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -14188,38 +14188,38 @@ define void @global_atomic_fsub_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg %ptr
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB261_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX90A-NEXT: v_sub_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s6
+; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v4
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v5
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s7
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB261_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -14364,38 +14364,38 @@ define void @global_atomic_fmax_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg %ptr
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB263_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX90A-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s6
+; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX90A-NEXT: v_max_f32_e32 v5, v5, v4
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v5
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s7
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB263_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -14540,38 +14540,38 @@ define void @global_atomic_fmin_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg %ptr
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB265_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX90A-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s6
+; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX90A-NEXT: v_min_f32_e32 v5, v5, v4
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v5
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s7
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB265_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -14716,43 +14716,43 @@ define void @global_atomic_fmaximum_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v4, 0x7fc00000
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB267_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_max_f32_e32 v6, v0, v3
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v0, v3
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX90A-NEXT: v_max_f32_e32 v7, v0, v3
-; GFX90A-NEXT: v_max_f32_e32 v8, v6, v5
+; GFX90A-NEXT: v_max_f32_e32 v7, v6, v5
; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v6, v5
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v0, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v4, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v4, v8, vcc
-; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v4, v7, vcc
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s7
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB267_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -14902,43 +14902,43 @@ define void @global_atomic_fminimum_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v4, 0x7fc00000
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB269_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_min_f32_e32 v6, v0, v3
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v0, v3
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX90A-NEXT: v_min_f32_e32 v7, v0, v3
-; GFX90A-NEXT: v_min_f32_e32 v8, v6, v5
+; GFX90A-NEXT: v_min_f32_e32 v7, v6, v5
; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v6, v5
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v0, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v4, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v4, v8, vcc
-; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v4, v7, vcc
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s7
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB269_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
diff --git a/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll b/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll
index 59f7e5c2b43f7c..431416e6f917a8 100644
--- a/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll
@@ -775,7 +775,7 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX90A-NEXT: s_mov_b64 s[4:5], -1
; GFX90A-NEXT: s_xor_b64 s[6:7], s[6:7], s[4:5]
; GFX90A-NEXT: s_and_b64 vcc, exec, s[6:7]
-; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX90A-NEXT: s_cbranch_vccnz .LBB5_3
; GFX90A-NEXT: .LBB5_1: ; %Flow4
; GFX90A-NEXT: s_mov_b64 s[6:7], -1
@@ -783,7 +783,7 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX90A-NEXT: s_and_b64 vcc, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_vccnz .LBB5_10
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.shared
-; GFX90A-NEXT: ds_add_rtn_f64 v[2:3], v0, v[0:1]
+; GFX90A-NEXT: ds_add_rtn_f64 v[4:5], v0, v[0:1]
; GFX90A-NEXT: s_branch .LBB5_10
; GFX90A-NEXT: .LBB5_3: ; %atomicrmw.check.private
; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base
@@ -795,16 +795,16 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX90A-NEXT: s_mov_b64 s[4:5], -1
; GFX90A-NEXT: s_xor_b64 s[6:7], s[6:7], s[4:5]
; GFX90A-NEXT: s_and_b64 vcc, exec, s[6:7]
-; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX90A-NEXT: s_cbranch_vccnz .LBB5_5
; GFX90A-NEXT: s_branch .LBB5_6
; GFX90A-NEXT: .LBB5_4: ; %atomicrmw.private
-; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
+; GFX90A-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: v_add_f64 v[4:5], v[2:3], v[0:1]
-; GFX90A-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen
-; GFX90A-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen
+; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1]
+; GFX90A-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
+; GFX90A-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GFX90A-NEXT: s_branch .LBB5_9
; GFX90A-NEXT: .LBB5_5: ; %atomicrmw.global
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
@@ -829,8 +829,8 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX90A-NEXT: global_atomic_cmpswap_x2 v[2:3], v6, v[2:5], s[6:7] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u64_e64 s[6:7], v[2:3], v[4:5]
-; GFX90A-NEXT: s_or_b64 s[4:5], s[6:7], s[4:5]
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
+; GFX90A-NEXT: s_or_b64 s[4:5], s[6:7], s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB5_7
; GFX90A-NEXT: ; %bb.8: ; %atomicrmw.end1
@@ -844,9 +844,9 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX90A-NEXT: ; %bb.11: ; %atomicrmw.end
; GFX90A-NEXT: s_mov_b32 s4, 32
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b64 v[4:5], s4, v[2:3]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v2
-; GFX90A-NEXT: v_mov_b32_e32 v1, v4
+; GFX90A-NEXT: v_lshrrev_b64 v[2:3], s4, v[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v4
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -862,7 +862,7 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX942-NEXT: s_mov_b64 s[0:1], -1
; GFX942-NEXT: s_xor_b64 s[2:3], s[2:3], s[0:1]
; GFX942-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX942-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX942-NEXT: s_cbranch_vccnz .LBB5_3
; GFX942-NEXT: .LBB5_1: ; %Flow4
; GFX942-NEXT: s_mov_b64 s[2:3], -1
@@ -870,7 +870,7 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX942-NEXT: s_and_b64 vcc, exec, s[0:1]
; GFX942-NEXT: s_cbranch_vccnz .LBB5_10
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.shared
-; GFX942-NEXT: ds_add_rtn_f64 v[2:3], v0, v[0:1]
+; GFX942-NEXT: ds_add_rtn_f64 v[4:5], v0, v[0:1]
; GFX942-NEXT: s_branch .LBB5_10
; GFX942-NEXT: .LBB5_3: ; %atomicrmw.check.private
; GFX942-NEXT: s_mov_b64 s[0:1], src_private_base
@@ -882,14 +882,14 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX942-NEXT: s_mov_b64 s[0:1], -1
; GFX942-NEXT: s_xor_b64 s[2:3], s[2:3], s[0:1]
; GFX942-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX942-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX942-NEXT: s_cbranch_vccnz .LBB5_5
; GFX942-NEXT: s_branch .LBB5_6
; GFX942-NEXT: .LBB5_4: ; %atomicrmw.private
-; GFX942-NEXT: scratch_load_dwordx2 v[2:3], off, s0
+; GFX942-NEXT: scratch_load_dwordx2 v[4:5], off, s0
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_add_f64 v[4:5], v[2:3], v[0:1]
-; GFX942-NEXT: scratch_store_dwordx2 off, v[4:5], s0
+; GFX942-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1]
+; GFX942-NEXT: scratch_store_dwordx2 off, v[2:3], s0
; GFX942-NEXT: s_branch .LBB5_9
; GFX942-NEXT: .LBB5_5: ; %atomicrmw.global
; GFX942-NEXT: v_mov_b32_e32 v2, 0
@@ -914,8 +914,8 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX942-NEXT: global_atomic_cmpswap_x2 v[2:3], v6, v[2:5], s[2:3] sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_cmp_eq_u64_e64 s[2:3], v[2:3], v[4:5]
-; GFX942-NEXT: s_or_b64 s[0:1], s[2:3], s[0:1]
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
+; GFX942-NEXT: s_or_b64 s[0:1], s[2:3], s[0:1]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB5_7
; GFX942-NEXT: ; %bb.8: ; %atomicrmw.end1
@@ -929,9 +929,9 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX942-NEXT: ; %bb.11: ; %atomicrmw.end
; GFX942-NEXT: s_mov_b32 s0, 32
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshrrev_b64 v[4:5], s0, v[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v2
-; GFX942-NEXT: v_mov_b32_e32 v1, v4
+; GFX942-NEXT: v_lshrrev_b64 v[2:3], s0, v[4:5]
+; GFX942-NEXT: v_mov_b32_e32 v0, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll b/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
index fbcba25ba207cf..1d4ada34650c3a 100644
--- a/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
@@ -7,14 +7,16 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: s_load_dword s0, s[4:5], 0x8
; CHECK-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x0
; CHECK-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x10
-; CHECK-NEXT: v_mov_b32_e32 v30, 0x9037ab78
-; CHECK-NEXT: v_mov_b32_e32 v31, 0x3e21eeb6
+; CHECK-NEXT: v_mov_b32_e32 v0, 0x9037ab78
+; CHECK-NEXT: v_mov_b32_e32 v1, 0x3e21eeb6
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_bitcmp1_b32 s0, 0
; CHECK-NEXT: s_cselect_b64 s[14:15], -1, 0
; CHECK-NEXT: s_xor_b64 s[16:17], s[14:15], -1
; CHECK-NEXT: s_bitcmp1_b32 s0, 8
+; CHECK-NEXT: v_accvgpr_write_b32 a4, v1 ; Reload Reuse
; CHECK-NEXT: s_cselect_b64 s[18:19], -1, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a5, v0 ; Reload Reuse
; CHECK-NEXT: s_xor_b64 s[20:21], s[18:19], -1
; CHECK-NEXT: v_mov_b32_e32 v2, 0xa17f65f6
; CHECK-NEXT: v_mov_b32_e32 v3, 0xbe927e4f
@@ -32,14 +34,14 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: v_mov_b32_e32 v14, 0x8427b883
; CHECK-NEXT: v_mov_b32_e32 v15, 0x3fae1bb4
; CHECK-NEXT: s_and_b64 s[0:1], exec, s[14:15]
-; CHECK-NEXT: v_mov_b32_e32 v16, 0x57b87036
-; CHECK-NEXT: v_mov_b32_e32 v17, 0x3fb3b136
+; CHECK-NEXT: v_mov_b32_e32 v20, 0x57b87036
+; CHECK-NEXT: v_mov_b32_e32 v21, 0x3fb3b136
; CHECK-NEXT: s_and_b64 s[2:3], exec, s[18:19]
; CHECK-NEXT: v_mov_b32_e32 v18, 0x55555523
; CHECK-NEXT: v_mov_b32_e32 v19, 0xbfd55555
; CHECK-NEXT: s_and_b64 s[4:5], exec, s[16:17]
; CHECK-NEXT: v_mov_b32_e32 v0, 0
-; CHECK-NEXT: v_mov_b64_e32 v[20:21], 0
+; CHECK-NEXT: v_mov_b64_e32 v[30:31], 0
; CHECK-NEXT: ; implicit-def: $agpr0_agpr1
; CHECK-NEXT: ; implicit-def: $vgpr22_vgpr23
; CHECK-NEXT: s_branch .LBB0_2
@@ -64,8 +66,11 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: v_mov_b64_e32 v[24:25], s[10:11]
; CHECK-NEXT: flat_load_dwordx2 v[24:25], v[24:25]
-; CHECK-NEXT: v_mov_b64_e32 v[26:27], v[30:31]
+; CHECK-NEXT: v_accvgpr_read_b32 v27, a4 ; Reload Reuse
+; CHECK-NEXT: v_accvgpr_read_b32 v26, a5 ; Reload Reuse
; CHECK-NEXT: v_mov_b64_e32 v[28:29], v[2:3]
+; CHECK-NEXT: v_mov_b64_e32 v[16:17], v[20:21]
+; CHECK-NEXT: v_mov_b64_e32 v[30:31], 0
; CHECK-NEXT: v_accvgpr_write_b32 a2, 0
; CHECK-NEXT: v_accvgpr_write_b32 a3, 0
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -83,10 +88,9 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: v_fmac_f64_e32 v[26:27], 0, v[28:29]
; CHECK-NEXT: v_mov_b64_e32 v[28:29], v[14:15]
; CHECK-NEXT: v_fmac_f64_e32 v[28:29], 0, v[26:27]
-; CHECK-NEXT: v_mov_b64_e32 v[26:27], v[16:17]
-; CHECK-NEXT: v_fmac_f64_e32 v[26:27], 0, v[28:29]
-; CHECK-NEXT: v_mov_b64_e32 v[28:29], v[18:19]
-; CHECK-NEXT: v_fmac_f64_e32 v[28:29], 0, v[26:27]
+; CHECK-NEXT: v_fmac_f64_e32 v[16:17], 0, v[28:29]
+; CHECK-NEXT: v_mov_b64_e32 v[26:27], v[18:19]
+; CHECK-NEXT: v_fmac_f64_e32 v[26:27], 0, v[16:17]
; CHECK-NEXT: s_branch .LBB0_6
; CHECK-NEXT: .LBB0_5: ; %Flow
; CHECK-NEXT: ; in Loop: Header=BB0_6 Depth=2
@@ -95,8 +99,8 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: .LBB0_6: ; %.preheader1855.i.i.i3329
; CHECK-NEXT: ; Parent Loop BB0_2 Depth=1
; CHECK-NEXT: ; => This Inner Loop Header: Depth=2
-; CHECK-NEXT: v_accvgpr_read_b32 v27, a3
-; CHECK-NEXT: v_accvgpr_read_b32 v26, a2
+; CHECK-NEXT: v_accvgpr_read_b32 v29, a3
+; CHECK-NEXT: v_accvgpr_read_b32 v28, a2
; CHECK-NEXT: s_mov_b64 s[24:25], -1
; CHECK-NEXT: s_mov_b64 s[6:7], -1
; CHECK-NEXT: s_mov_b64 vcc, s[2:3]
@@ -112,8 +116,8 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: ; %bb.8: ; %.preheader1856.preheader.i.i.i3325
; CHECK-NEXT: ; in Loop: Header=BB0_6 Depth=2
; CHECK-NEXT: s_mov_b64 s[24:25], 0
-; CHECK-NEXT: v_accvgpr_write_b32 a2, v28
-; CHECK-NEXT: v_accvgpr_write_b32 a3, v29
+; CHECK-NEXT: v_accvgpr_write_b32 a2, v26
+; CHECK-NEXT: v_accvgpr_write_b32 a3, v27
; CHECK-NEXT: s_mov_b64 s[6:7], 0
; CHECK-NEXT: s_branch .LBB0_5
; CHECK-NEXT: .LBB0_9: ; in Loop: Header=BB0_2 Depth=1
@@ -133,16 +137,17 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: s_cbranch_vccz .LBB0_13
; CHECK-NEXT: ; %bb.12: ; %._crit_edge2105.i.i.i2330.loopexit
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: v_cmp_nlg_f64_e64 s[6:7], 0, v[26:27]
+; CHECK-NEXT: v_cmp_nlg_f64_e64 s[6:7], 0, v[28:29]
; CHECK-NEXT: v_cndmask_b32_e64 v23, v23, 0, s[14:15]
; CHECK-NEXT: v_cndmask_b32_e64 v22, v22, 0, s[14:15]
-; CHECK-NEXT: v_cndmask_b32_e64 v26, 0, 1, s[6:7]
-; CHECK-NEXT: v_mov_b32_e32 v27, v26
+; CHECK-NEXT: v_cndmask_b32_e64 v16, 0, 1, s[6:7]
+; CHECK-NEXT: v_mov_b32_e32 v17, v16
; CHECK-NEXT: s_and_b64 s[6:7], exec, s[14:15]
-; CHECK-NEXT: global_store_dwordx2 v0, v[26:27], s[8:9]
+; CHECK-NEXT: global_store_dwordx2 v0, v[16:17], s[8:9]
; CHECK-NEXT: s_cselect_b32 s23, s23, 0
; CHECK-NEXT: s_cselect_b32 s22, s22, 0
; CHECK-NEXT: s_mov_b64 s[6:7], -1
+; CHECK-NEXT: v_mov_b64_e32 v[30:31], 0
; CHECK-NEXT: s_branch .LBB0_14
; CHECK-NEXT: .LBB0_13: ; in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: s_mov_b64 s[6:7], 0
@@ -159,7 +164,7 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: ; %bb.16: ; %._crit_edge2105.i.i.i2330
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: s_mov_b64 s[24:25], 0
-; CHECK-NEXT: global_store_dwordx2 v0, v[20:21], s[8:9]
+; CHECK-NEXT: global_store_dwordx2 v0, v[30:31], s[8:9]
; CHECK-NEXT: s_branch .LBB0_1
; CHECK-NEXT: .LBB0_17: ; %DummyReturnBlock
; CHECK-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/copy-to-reg-frameindex.ll b/llvm/test/CodeGen/AMDGPU/copy-to-reg-frameindex.ll
index f7e9b8724b6821..04c40e7ccdaf6b 100644
--- a/llvm/test/CodeGen/AMDGPU/copy-to-reg-frameindex.ll
+++ b/llvm/test/CodeGen/AMDGPU/copy-to-reg-frameindex.ll
@@ -43,10 +43,11 @@ define void @phi_with_alloca_and_divergent_copy_to_reg(ptr addrspace(5) %diverge
; CHECK-LABEL: phi_with_alloca_and_divergent_copy_to_reg:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_lshr_b32 s6, s32, 6
; CHECK-NEXT: v_mov_b32_e32 v7, v2
; CHECK-NEXT: v_mov_b32_e32 v6, v1
; CHECK-NEXT: s_mov_b64 s[4:5], 0
-; CHECK-NEXT: v_lshrrev_b32_e64 v2, 6, s32
+; CHECK-NEXT: v_mov_b32_e32 v2, s6
; CHECK-NEXT: .LBB1_1: ; %loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: v_mov_b32_e32 v1, v2
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
index f40b209b77da7a..bb2dc353ab9fa5 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
@@ -14644,19 +14644,20 @@ define void @flat_atomic_fmax_f32_saddr_nortn(ptr inreg %ptr, float %data) {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[2:3] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: v_max_f32_e32 v4, v0, v0
+; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: .LBB119_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX950-NEXT: v_max_f32_e32 v0, v0, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX950-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB119_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14720,19 +14721,20 @@ define void @flat_atomic_fmin_f32_saddr_nortn(ptr inreg %ptr, float %data) {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[2:3] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: v_max_f32_e32 v4, v0, v0
+; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: .LBB121_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX950-NEXT: v_min_f32_e32 v0, v0, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX950-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB121_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14894,21 +14896,22 @@ define void @flat_atomic_fmax_v2f16_saddr_nortn(ptr inreg %ptr, <2 x half> %data
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[2:3] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: v_pk_max_f16 v4, v0, v0
+; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB125_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX950-NEXT: v_pk_max_f16 v2, v3, v3
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_max_f16 v0, v0, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_pk_max_f16 v2, v2, v4
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB125_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -15006,21 +15009,22 @@ define void @flat_atomic_fmin_v2f16_saddr_nortn(ptr inreg %ptr, <2 x half> %data
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[2:3] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: v_pk_max_f16 v4, v0, v0
+; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB127_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX950-NEXT: v_pk_max_f16 v2, v3, v3
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_min_f16 v0, v0, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_pk_min_f16 v2, v2, v4
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB127_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/insert-waitcnts-merge.ll b/llvm/test/CodeGen/AMDGPU/insert-waitcnts-merge.ll
index fadb15dd2fb799..03d1abcb053719 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-waitcnts-merge.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert-waitcnts-merge.ll
@@ -23,9 +23,9 @@ define amdgpu_kernel void @widget(ptr addrspace(1) %arg, i1 %arg1) #0 {
; CHECK-NEXT: S_WAITCNT .Lgkmcnt_0
; CHECK-NEXT: S_BITCMP1_B32 killed renamable $sgpr2, 0, implicit-def $scc
; CHECK-NEXT: renamable $sgpr4_sgpr5 = S_MOV_B64 0
- ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
; CHECK-NEXT: renamable $sgpr2_sgpr3 = S_CSELECT_B64 -1, 0, implicit killed $scc
; CHECK-NEXT: $vgpr2_vgpr3 = V_PK_MOV_B32 8, 0, 8, 0, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; CHECK-NEXT: renamable $sgpr6_sgpr7 = IMPLICIT_DEF
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
index e5e07ff1a94eb1..571c8aa6631c75 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
@@ -6430,19 +6430,19 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-LABEL: local_atomic_fadd_ret_v2bf16:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v3, v0
+; GFX90A-NEXT: ds_read_b32 v4, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX90A-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX90A-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_add_f32_e32 v5, v5, v3
; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6454,11 +6454,11 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v1
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6791,19 +6791,19 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-LABEL: local_atomic_fadd_ret_v2bf16__offset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v3, v0 offset:65532
+; GFX90A-NEXT: ds_read_b32 v4, v0 offset:65532
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX90A-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX90A-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_add_f32_e32 v5, v5, v3
; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6815,11 +6815,11 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1 offset:65532
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v1
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB25_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
index b74f6b1c346012..671a8c49025bba 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
@@ -6157,19 +6157,19 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX942-LABEL: local_atomic_fmax_ret_v2bf16:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: ds_read_b32 v3, v0
+; GFX942-NEXT: ds_read_b32 v4, v0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
-; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX942-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX942-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX942-NEXT: v_max_f32_e32 v5, v5, v3
; GFX942-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6182,11 +6182,11 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX942-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[0:1]
; GFX942-NEXT: v_perm_b32 v1, v5, v1, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v3, v1
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB24_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6329,19 +6329,19 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-LABEL: local_atomic_fmax_ret_v2bf16:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v3, v0
+; GFX90A-NEXT: ds_read_b32 v4, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX90A-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX90A-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_max_f32_e32 v5, v5, v3
; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6353,11 +6353,11 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v1
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6643,19 +6643,19 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX942-LABEL: local_atomic_fmax_ret_v2bf16__offset:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: ds_read_b32 v3, v0 offset:65532
+; GFX942-NEXT: ds_read_b32 v4, v0 offset:65532
; GFX942-NEXT: s_mov_b64 s[2:3], 0
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
-; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX942-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX942-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX942-NEXT: v_max_f32_e32 v5, v5, v3
; GFX942-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6668,11 +6668,11 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX942-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[0:1]
; GFX942-NEXT: v_perm_b32 v1, v5, v1, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1 offset:65532
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v3, v1
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB25_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6815,19 +6815,19 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-LABEL: local_atomic_fmax_ret_v2bf16__offset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v3, v0 offset:65532
+; GFX90A-NEXT: ds_read_b32 v4, v0 offset:65532
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX90A-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX90A-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_max_f32_e32 v5, v5, v3
; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6839,11 +6839,11 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1 offset:65532
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v1
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB25_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
index 7c68c45a70a038..e9f671c44cffbe 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
@@ -6157,19 +6157,19 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX942-LABEL: local_atomic_fmin_ret_v2bf16:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: ds_read_b32 v3, v0
+; GFX942-NEXT: ds_read_b32 v4, v0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
-; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX942-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX942-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX942-NEXT: v_min_f32_e32 v5, v5, v3
; GFX942-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6182,11 +6182,11 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX942-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[0:1]
; GFX942-NEXT: v_perm_b32 v1, v5, v1, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v3, v1
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB24_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6329,19 +6329,19 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-LABEL: local_atomic_fmin_ret_v2bf16:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v3, v0
+; GFX90A-NEXT: ds_read_b32 v4, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX90A-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX90A-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_min_f32_e32 v5, v5, v3
; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6353,11 +6353,11 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v1
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6643,19 +6643,19 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX942-LABEL: local_atomic_fmin_ret_v2bf16__offset:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: ds_read_b32 v3, v0 offset:65532
+; GFX942-NEXT: ds_read_b32 v4, v0 offset:65532
; GFX942-NEXT: s_mov_b64 s[2:3], 0
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
-; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX942-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX942-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX942-NEXT: v_min_f32_e32 v5, v5, v3
; GFX942-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6668,11 +6668,11 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX942-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[0:1]
; GFX942-NEXT: v_perm_b32 v1, v5, v1, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1 offset:65532
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v3, v1
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB25_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6815,19 +6815,19 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-LABEL: local_atomic_fmin_ret_v2bf16__offset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v3, v0 offset:65532
+; GFX90A-NEXT: ds_read_b32 v4, v0 offset:65532
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX90A-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX90A-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_min_f32_e32 v5, v5, v3
; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6839,11 +6839,11 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1 offset:65532
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v1
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB25_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
index 5c988fcda1a8e4..38723293fabb3d 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
@@ -6971,19 +6971,19 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX942-LABEL: local_atomic_fsub_ret_v2bf16:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: ds_read_b32 v3, v0
+; GFX942-NEXT: ds_read_b32 v4, v0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
-; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX942-NEXT: v_sub_f32_e32 v1, v1, v2
-; GFX942-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX942-NEXT: v_sub_f32_e32 v5, v5, v3
; GFX942-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6996,11 +6996,11 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX942-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[0:1]
; GFX942-NEXT: v_perm_b32 v1, v5, v1, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v3, v1
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB24_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7143,19 +7143,19 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-LABEL: local_atomic_fsub_ret_v2bf16:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v3, v0
+; GFX90A-NEXT: ds_read_b32 v4, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX90A-NEXT: v_sub_f32_e32 v1, v1, v2
-; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v3
; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -7167,11 +7167,11 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v1
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7457,19 +7457,19 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX942-LABEL: local_atomic_fsub_ret_v2bf16__offset:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: ds_read_b32 v3, v0 offset:65532
+; GFX942-NEXT: ds_read_b32 v4, v0 offset:65532
; GFX942-NEXT: s_mov_b64 s[2:3], 0
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
-; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX942-NEXT: v_sub_f32_e32 v1, v1, v2
-; GFX942-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX942-NEXT: v_sub_f32_e32 v5, v5, v3
; GFX942-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -7482,11 +7482,11 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX942-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[0:1]
; GFX942-NEXT: v_perm_b32 v1, v5, v1, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1 offset:65532
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v3, v1
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB25_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7629,19 +7629,19 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-LABEL: local_atomic_fsub_ret_v2bf16__offset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v3, v0 offset:65532
+; GFX90A-NEXT: ds_read_b32 v4, v0 offset:65532
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX90A-NEXT: v_sub_f32_e32 v1, v1, v2
-; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v3
; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -7653,11 +7653,11 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1 offset:65532
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v1
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB25_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/phi-av-pressure.ll b/llvm/test/CodeGen/AMDGPU/phi-av-pressure.ll
index a4bf3705534bc4..29f26bb00135b3 100644
--- a/llvm/test/CodeGen/AMDGPU/phi-av-pressure.ll
+++ b/llvm/test/CodeGen/AMDGPU/phi-av-pressure.ll
@@ -36,9 +36,9 @@ define amdgpu_kernel void @main(i1 %arg, ptr %ptr, ptr addrspace(1) %ptr1, ptr a
; GFX950-NEXT: v_lshl_add_u64 v[10:11], v[0:1], 3, s[38:39]
; GFX950-NEXT: global_load_dwordx4 v[2:5], v[10:11], off
; GFX950-NEXT: global_load_dwordx4 v[6:9], v1, s[38:39]
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
; GFX950-NEXT: v_mov_b64_e32 v[10:11], s[36:37]
; GFX950-NEXT: s_and_b64 vcc, exec, s[34:35]
+; GFX950-NEXT: v_mov_b32_e32 v0, v1
; GFX950-NEXT: v_mov_b32_e32 v12, v1
; GFX950-NEXT: v_mov_b32_e32 v14, v1
; GFX950-NEXT: v_mov_b32_e32 v15, v1
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-spill-cost-reset.ll b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-spill-cost-reset.ll
index d115f9bd028f27..1b2b93f1e12d4c 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-spill-cost-reset.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-spill-cost-reset.ll
@@ -68,86 +68,62 @@ define amdgpu_kernel void @test_spill_cost_reset(
; CHECK-NEXT: liveins: $sgpr4_sgpr5
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
- ; CHECK-NEXT: early-clobber %1891:sgpr_256 = S_LOAD_DWORDX8_IMM_ec [[COPY]](p4), 16, 0 :: (dereferenceable invariant load (s256) from %ir.a0.kernarg.offset, align 16, addrspace 4)
- ; CHECK-NEXT: early-clobber %1892:sgpr_256 = S_LOAD_DWORDX8_IMM_ec [[COPY]](p4), 48, 0 :: (dereferenceable invariant load (s256) from %ir.b0.kernarg.offset, align 16, addrspace 4)
- ; CHECK-NEXT: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]](p4), 80, 0 :: (dereferenceable invariant load (s32) from %ir.n.kernarg.offset, align 16, addrspace 4)
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: early-clobber %1843:sgpr_256 = S_LOAD_DWORDX8_IMM_ec [[COPY]](p4), 16, 0 :: (dereferenceable invariant load (s256) from %ir.a0.kernarg.offset, align 16, addrspace 4)
+ ; CHECK-NEXT: early-clobber %1844:sgpr_256 = S_LOAD_DWORDX8_IMM_ec [[COPY]](p4), 48, 0 :: (dereferenceable invariant load (s256) from %ir.b0.kernarg.offset, align 16, addrspace 4)
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_1:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_2:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_3:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_4:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_5:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_5:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_5:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_5:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_6:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_6:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_6:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_6:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_7:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_7:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_7:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_7:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_8:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_8:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_8:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_8:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_9:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_9:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_9:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_9:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_10:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_10:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_10:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_10:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_11:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_11:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_11:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_11:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_12:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_12:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_12:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_12:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_128_align2 = COPY %1891.sub0_sub1_sub2_sub3
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_128_align2 = COPY %1892.sub0_sub1_sub2_sub3
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:av_128_align2 = COPY %1891.sub4_sub5_sub6_sub7
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:av_128_align2 = COPY %1892.sub4_sub5_sub6_sub7
- ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY5:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY5:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY6:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY6:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY7:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY7:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY8:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY8:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY9:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY9:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY10:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY10:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY11:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY11:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY12:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY12:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY13:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY13:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY14:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY14:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY15:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY15:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]](p4), 80, 0 :: (dereferenceable invariant load (s32) from %ir.n.kernarg.offset, align 16, addrspace 4)
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_128_align2 = COPY %1843.sub0_sub1_sub2_sub3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_128_align2 = COPY %1844.sub0_sub1_sub2_sub3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:av_128_align2 = COPY %1843.sub4_sub5_sub6_sub7
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:av_128_align2 = COPY %1844.sub4_sub5_sub6_sub7
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY5:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY6:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY7:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY8:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY9:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY10:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY11:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY12:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY13:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY14:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY15:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
; CHECK-NEXT: undef [[COPY16:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
; CHECK-NEXT: [[COPY16:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
; CHECK-NEXT: undef [[COPY17:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
@@ -380,69 +356,105 @@ define amdgpu_kernel void @test_spill_cost_reset(
; CHECK-NEXT: [[COPY130:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
; CHECK-NEXT: undef [[COPY131:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
; CHECK-NEXT: [[COPY131:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY132:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY132:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY133:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY133:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY134:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY134:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY135:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY135:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY136:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY136:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY137:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY137:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY138:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY138:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY139:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY139:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY140:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY140:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY141:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY141:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY142:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY142:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY143:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY143:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1.loop:
; CHECK-NEXT: successors: %bb.2(0x04000000), %bb.1(0x7c000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[V_MOV_B32_e32_12:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_12]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY15]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY143:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY143]], 8, [[COPY143]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY142:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY142]], 8, [[COPY142]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY141:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY141]], 8, [[COPY141]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY140:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY140]], 8, [[COPY140]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY139:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY139]], 8, [[COPY139]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY14]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY138:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY138]], 8, [[COPY138]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY137:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY137]], 8, [[COPY137]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY136:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY136]], 8, [[COPY136]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY13]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY135:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY135]], 8, [[COPY135]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY134:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY134]], 8, [[COPY134]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY133:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY133]], 8, [[COPY133]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY12]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY132:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY132]], 8, [[COPY132]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY131:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY131]], 8, [[COPY131]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY130:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY130]], 8, [[COPY130]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY11]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY129:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY129]], 8, [[COPY129]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY128:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY128]], 8, [[COPY128]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY127:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY127]], 8, [[COPY127]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_11:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_11]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY10]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY126:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY126]], 8, [[COPY126]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY125:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY125]], 8, [[COPY125]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY124:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY124]], 8, [[COPY124]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_10:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_10]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY9]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY123:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY123]], 8, [[COPY123]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY122:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY122]], 8, [[COPY122]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY121:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY121]], 8, [[COPY121]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_9:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_9]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY8]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY120:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY120]], 8, [[COPY120]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY119:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY119]], 8, [[COPY119]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY118:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY118]], 8, [[COPY118]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_8:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_8]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY7]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY117:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY117]], 8, [[COPY117]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY116:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY116]], 8, [[COPY116]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY115:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY115]], 8, [[COPY115]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_7:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_7]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY6]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY114:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY114]], 8, [[COPY114]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY113:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY113]], 8, [[COPY113]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY112:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY112]], 8, [[COPY112]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_6:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_6]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY5]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY111:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY111]], 8, [[COPY111]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY110:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY110]], 8, [[COPY110]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY109:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY109]], 8, [[COPY109]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_5:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_5]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY108:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY108]], 8, [[COPY108]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY107:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY107]], 8, [[COPY107]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY106:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY106]], 8, [[COPY106]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_4]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY105:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY105]], 8, [[COPY105]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY104:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY104]], 8, [[COPY104]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY103:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY103]], 8, [[COPY103]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_3]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY3]], [[COPY4]], [[COPY15]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_1:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY102:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY102]], 8, [[COPY102]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY101:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY101]], 8, [[COPY101]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY100:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY100]], 8, [[COPY100]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_2]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_2:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY3]], [[COPY4]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY99:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY99]], 8, [[COPY99]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY98:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY98]], 8, [[COPY98]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY97:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY97]], 8, [[COPY97]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_1]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY96:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY96]], 8, [[COPY96]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY95:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY95]], 8, [[COPY95]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY94:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY94]], 8, [[COPY94]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY93:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY93]], 8, [[COPY93]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY92:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY92]], 8, [[COPY92]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY91:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY91]], 8, [[COPY91]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY3]], [[COPY4]], [[V_MOV_B32_e32_12]], 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_1:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY90:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY90]], 8, [[COPY90]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY89:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY89]], 8, [[COPY89]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY88:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY88]], 8, [[COPY88]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_2:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY3]], [[COPY4]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY87:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY87]], 8, [[COPY87]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY86:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY86]], 8, [[COPY86]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY85:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY85]], 8, [[COPY85]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
@@ -514,28 +526,16 @@ define amdgpu_kernel void @test_spill_cost_reset(
; CHECK-NEXT: [[COPY19:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY19]], 8, [[COPY19]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY18:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY18]], 8, [[COPY18]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY17:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY17]], 8, [[COPY17]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY16:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY16]], 8, [[COPY16]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY15]], 8, [[COPY15]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY14]], 8, [[COPY14]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY13]], 8, [[COPY13]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY12]], 8, [[COPY12]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY11]], 8, [[COPY11]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY10]], 8, [[COPY10]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY9]], 8, [[COPY9]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY8]], 8, [[COPY8]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY7]], 8, [[COPY7]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY6]], 8, [[COPY6]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY5]], 8, [[COPY5]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_MOV_B32_]], 1, implicit-def dead $scc
; CHECK-NEXT: S_CMP_LT_I32 [[S_MOV_B32_]], [[S_LOAD_DWORD_IMM]], implicit-def $scc
- ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_]], 8, [[V_MOV_B32_e32_]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY16:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY16]], 8, [[COPY16]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: S_CBRANCH_SCC1 %bb.1, implicit killed $scc
; CHECK-NEXT: S_BRANCH %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2.epilogue:
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 0, 0 :: (dereferenceable invariant load (s64) from %ir.out.kernarg.offset550, align 16, addrspace 4)
- ; CHECK-NEXT: [[V_MOV_B32_e32_13:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: GLOBAL_STORE_DWORD_SADDR [[V_MOV_B32_e32_13]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_2]].sub0, [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s32) into %ir.out.load, addrspace 1)
+ ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD_SADDR [[V_MOV_B32_e32_1]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_2]].sub0, [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s32) into %ir.out.load, addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
ptr addrspace(1) %out,
<8 x half> %a0, <8 x half> %a1,
diff --git a/llvm/test/CodeGen/AMDGPU/si-fix-sgpr-copies-phi-av-classes.mir b/llvm/test/CodeGen/AMDGPU/si-fix-sgpr-copies-phi-av-classes.mir
new file mode 100644
index 00000000000000..ee2974411e328f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/si-fix-sgpr-copies-phi-av-classes.mir
@@ -0,0 +1,386 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu9.0a -run-pass=si-fix-sgpr-copies -verify-machineinstrs -o - %s | FileCheck %s
+
+# PHI operand legalization must not insert copies between vector register
+# classes that only differ in AV-ness when their common subclass is a VGPR
+# class (e.g. av_32 vs. vgpr_32). Copies from SGPRs, between VGPR and AGPR
+# classes, and between AV and AGPR classes are still required.
+
+---
+name: phi_vgpr_av_operand
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: phi_vgpr_av_operand
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0, $vgpr1, $sgpr0_sgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:av_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+ ; CHECK-NEXT: $vcc = COPY [[COPY2]]
+ ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[COPY1]], [[COPY1]], implicit $exec
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY]], %bb.0, [[V_ADD_U32_e32_]], %bb.1
+ ; CHECK-NEXT: $vgpr0 = COPY [[PHI]]
+ ; CHECK-NEXT: SI_RETURN implicit $vgpr0
+ bb.0:
+ successors: %bb.1, %bb.2
+ liveins: $vgpr0, $vgpr1, $sgpr0_sgpr1
+
+ %0:av_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:sreg_64 = COPY $sgpr0_sgpr1
+ $vcc = COPY %2
+ S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2
+ %3:vgpr_32 = V_ADD_U32_e32 %1, %1, implicit $exec
+
+ bb.2:
+ %4:vgpr_32 = PHI %0, %bb.0, %3, %bb.1
+ $vgpr0 = COPY %4
+ SI_RETURN implicit $vgpr0
+...
+
+---
+name: phi_av_vgpr_operand
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: phi_av_vgpr_operand
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0, $vgpr1, $sgpr0_sgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_32 = COPY $vgpr1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+ ; CHECK-NEXT: $vcc = COPY [[COPY2]]
+ ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:av_32 = COPY [[COPY1]]
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:av_32 = PHI [[COPY]], %bb.0, [[COPY3]], %bb.1
+ ; CHECK-NEXT: $vgpr0 = COPY [[PHI]]
+ ; CHECK-NEXT: SI_RETURN implicit $vgpr0
+ bb.0:
+ successors: %bb.1, %bb.2
+ liveins: $vgpr0, $vgpr1, $sgpr0_sgpr1
+
+ %0:vgpr_32 = COPY $vgpr0
+ %1:av_32 = COPY $vgpr1
+ %2:sreg_64 = COPY $sgpr0_sgpr1
+ $vcc = COPY %2
+ S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2
+ %3:av_32 = COPY %1
+
+ bb.2:
+ %4:av_32 = PHI %0, %bb.0, %3, %bb.1
+ $vgpr0 = COPY %4
+ SI_RETURN implicit $vgpr0
+...
+
+---
+name: phi_av64_vreg64_operand
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: phi_av64_vreg64_operand
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $sgpr0_sgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+ ; CHECK-NEXT: $vcc = COPY [[COPY2]]
+ ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:av_64_align2 = COPY [[COPY1]]
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:av_64_align2 = PHI [[COPY]], %bb.0, [[COPY3]], %bb.1
+ ; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[PHI]]
+ ; CHECK-NEXT: SI_RETURN implicit $vgpr0_vgpr1
+ bb.0:
+ successors: %bb.1, %bb.2
+ liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $sgpr0_sgpr1
+
+ %0:vreg_64_align2 = COPY $vgpr0_vgpr1
+ %1:av_64_align2 = COPY $vgpr2_vgpr3
+ %2:sreg_64 = COPY $sgpr0_sgpr1
+ $vcc = COPY %2
+ S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2
+ %3:av_64_align2 = COPY %1
+
+ bb.2:
+ %4:av_64_align2 = PHI %0, %bb.0, %3, %bb.1
+ $vgpr0_vgpr1 = COPY %4
+ SI_RETURN implicit $vgpr0_vgpr1
+...
+
+# The SGPR operand still needs a copy to a vector register (see #168761).
+---
+name: phi_av_sgpr_operand
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: phi_av_sgpr_operand
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0, $sgpr0_sgpr1, $sgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+ ; CHECK-NEXT: $vcc = COPY [[COPY2]]
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:av_32 = COPY [[COPY]], implicit $exec
+ ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:av_32 = COPY [[COPY1]]
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:av_32 = PHI [[COPY3]], %bb.0, [[COPY4]], %bb.1
+ ; CHECK-NEXT: $vgpr0 = COPY [[PHI]]
+ ; CHECK-NEXT: SI_RETURN implicit $vgpr0
+ bb.0:
+ successors: %bb.1, %bb.2
+ liveins: $vgpr0, $sgpr0_sgpr1, $sgpr2
+
+ %0:sreg_32 = COPY $sgpr2
+ %1:av_32 = COPY $vgpr0
+ %2:sreg_64 = COPY $sgpr0_sgpr1
+ $vcc = COPY %2
+ S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2
+ %3:av_32 = COPY %1
+
+ bb.2:
+ %4:av_32 = PHI %0, %bb.0, %3, %bb.1
+ $vgpr0 = COPY %4
+ SI_RETURN implicit $vgpr0
+...
+
+# An AGPR operand of a VGPR PHI still needs a copy.
+---
+name: phi_vgpr_agpr_operand
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: phi_vgpr_agpr_operand
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0, $agpr0, $sgpr0_sgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:agpr_32 = COPY $agpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+ ; CHECK-NEXT: $vcc = COPY [[COPY2]]
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]], implicit $exec
+ ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[COPY1]], [[COPY1]], implicit $exec
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, [[V_ADD_U32_e32_]], %bb.1
+ ; CHECK-NEXT: $vgpr0 = COPY [[PHI]]
+ ; CHECK-NEXT: SI_RETURN implicit $vgpr0
+ bb.0:
+ successors: %bb.1, %bb.2
+ liveins: $vgpr0, $agpr0, $sgpr0_sgpr1
+
+ %0:agpr_32 = COPY $agpr0
+ %1:vgpr_32 = COPY $vgpr0
+ %2:sreg_64 = COPY $sgpr0_sgpr1
+ $vcc = COPY %2
+ S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2
+ %3:vgpr_32 = V_ADD_U32_e32 %1, %1, implicit $exec
+
+ bb.2:
+ %4:vgpr_32 = PHI %0, %bb.0, %3, %bb.1
+ $vgpr0 = COPY %4
+ SI_RETURN implicit $vgpr0
+...
+
+# A subregister operand keeps the copy.
+---
+name: phi_vgpr_av64_subreg_operand
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: phi_vgpr_av64_subreg_operand
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0_sgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+ ; CHECK-NEXT: $vcc = COPY [[COPY2]]
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1, implicit $exec
+ ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[COPY1]], [[COPY1]], implicit $exec
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, [[V_ADD_U32_e32_]], %bb.1
+ ; CHECK-NEXT: $vgpr0 = COPY [[PHI]]
+ ; CHECK-NEXT: SI_RETURN implicit $vgpr0
+ bb.0:
+ successors: %bb.1, %bb.2
+ liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0_sgpr1
+
+ %0:av_64_align2 = COPY $vgpr0_vgpr1
+ %1:vgpr_32 = COPY $vgpr2
+ %2:sreg_64 = COPY $sgpr0_sgpr1
+ $vcc = COPY %2
+ S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2
+ %3:vgpr_32 = V_ADD_U32_e32 %1, %1, implicit $exec
+
+ bb.2:
+ %4:vgpr_32 = PHI %0.sub1, %bb.0, %3, %bb.1
+ $vgpr0 = COPY %4
+ SI_RETURN implicit $vgpr0
+...
+
+# The common subclass of an av_32 operand and an agpr_32 PHI is an AGPR class.
+# The copy is kept.
+---
+name: phi_agpr_av_operand
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: phi_agpr_av_operand
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0, $agpr0, $sgpr0_sgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:av_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:agpr_32 = COPY $agpr0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+ ; CHECK-NEXT: $vcc = COPY [[COPY2]]
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr_32 = COPY [[COPY]], implicit $exec
+ ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:agpr_32 = COPY [[COPY1]]
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:agpr_32 = PHI [[COPY3]], %bb.0, [[COPY4]], %bb.1
+ ; CHECK-NEXT: $agpr0 = COPY [[PHI]]
+ ; CHECK-NEXT: SI_RETURN implicit $agpr0
+ bb.0:
+ successors: %bb.1, %bb.2
+ liveins: $vgpr0, $agpr0, $sgpr0_sgpr1
+
+ %0:av_32 = COPY $vgpr0
+ %1:agpr_32 = COPY $agpr0
+ %2:sreg_64 = COPY $sgpr0_sgpr1
+ $vcc = COPY %2
+ S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2
+ %3:agpr_32 = COPY %1
+
+ bb.2:
+ %4:agpr_32 = PHI %0, %bb.0, %3, %bb.1
+ $agpr0 = COPY %4
+ SI_RETURN implicit $agpr0
+...
+
+# The same with an agpr_32 operand for an av_32 PHI. The copy is kept.
+---
+name: phi_av_agpr_operand
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: phi_av_agpr_operand
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0, $agpr0, $sgpr0_sgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:agpr_32 = COPY $agpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+ ; CHECK-NEXT: $vcc = COPY [[COPY2]]
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:av_32 = COPY [[COPY]], implicit $exec
+ ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:av_32 = COPY [[COPY1]]
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:av_32 = PHI [[COPY3]], %bb.0, [[COPY4]], %bb.1
+ ; CHECK-NEXT: $vgpr0 = COPY [[PHI]]
+ ; CHECK-NEXT: SI_RETURN implicit $vgpr0
+ bb.0:
+ successors: %bb.1, %bb.2
+ liveins: $vgpr0, $agpr0, $sgpr0_sgpr1
+
+ %0:agpr_32 = COPY $agpr0
+ %1:av_32 = COPY $vgpr0
+ %2:sreg_64 = COPY $sgpr0_sgpr1
+ $vcc = COPY %2
+ S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2
+ %3:av_32 = COPY %1
+
+ bb.2:
+ %4:av_32 = PHI %0, %bb.0, %3, %bb.1
+ $vgpr0 = COPY %4
+ SI_RETURN implicit $vgpr0
+...
>From d4d9024f85769cae4def3bcf29dceb11bb389329 Mon Sep 17 00:00:00 2001
From: Pablo Reble <pablo.reble at amd.com>
Date: Fri, 25 Sep 2026 14:50:00 -0500
Subject: [PATCH 2/3] revert changes before rework
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 11 -
.../test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll | 773 +++++++++---------
.../CodeGen/AMDGPU/a-v-global-atomicrmw.ll | 504 ++++++------
llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll | 44 +-
.../AMDGPU/av-split-dead-valno-crash.ll | 43 +-
.../CodeGen/AMDGPU/copy-to-reg-frameindex.ll | 3 +-
.../test/CodeGen/AMDGPU/flat-saddr-atomics.ll | 52 +-
.../CodeGen/AMDGPU/insert-waitcnts-merge.ll | 2 +-
.../CodeGen/AMDGPU/local-atomicrmw-fadd.ll | 32 +-
.../CodeGen/AMDGPU/local-atomicrmw-fmax.ll | 64 +-
.../CodeGen/AMDGPU/local-atomicrmw-fmin.ll | 64 +-
.../CodeGen/AMDGPU/local-atomicrmw-fsub.ll | 64 +-
llvm/test/CodeGen/AMDGPU/phi-av-pressure.ll | 2 +-
.../rewrite-mfma-form-spill-cost-reset.ll | 218 ++---
.../si-fix-sgpr-copies-phi-av-classes.mir | 386 ---------
15 files changed, 935 insertions(+), 1327 deletions(-)
delete mode 100644 llvm/test/CodeGen/AMDGPU/si-fix-sgpr-copies-phi-av-classes.mir
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index de141a67e5c5e8..16ee5f56986804 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7765,17 +7765,6 @@ SIInstrInfo::legalizeOperands(MachineInstr &MI,
if (!Op.isReg() || !Op.getReg().isVirtual())
continue;
- // No copy is needed if the classes differ only in AV-ness, e.g. an av_32
- // incoming value for a vgpr_32 PHI.
- const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg());
- if (!Op.getSubReg() && VRC != &AMDGPU::VReg_1RegClass &&
- OpRC != &AMDGPU::VReg_1RegClass &&
- (RI.isVectorSuperClass(OpRC) || RI.isVectorSuperClass(VRC))) {
- const TargetRegisterClass *Common = RI.getCommonSubClass(OpRC, VRC);
- if (Common && RI.isVGPRClass(Common))
- continue;
- }
-
// MI is a PHI instruction.
MachineBasicBlock *InsertBB = MI.getOperand(I + 1).getMBB();
MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator();
diff --git a/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll b/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll
index 0d32dd3fe5d268..cb59bb925aab0b 100644
--- a/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll
+++ b/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll
@@ -11812,27 +11812,27 @@ define void @flat_atomic_fmaximum_v2f16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
-; GFX90A-NEXT: s_mov_b32 s6, 0x5040100
+; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: .LBB147_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v2, v3, v4
-; GFX90A-NEXT: v_cmp_o_f16_e32 vcc, v3, v4
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v2, vcc
; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v3, v4 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v3, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v2, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_sdwa v2, v5, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s6
+; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s8
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB147_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -11950,27 +11950,27 @@ define void @flat_atomic_fminimum_v2f16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
-; GFX90A-NEXT: s_mov_b32 s6, 0x5040100
+; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: .LBB149_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_min_f16 v2, v3, v4
-; GFX90A-NEXT: v_cmp_o_f16_e32 vcc, v3, v4
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v2, vcc
; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v3, v4 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v3, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v2, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_sdwa v2, v5, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s6
+; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s8
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB149_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -12092,38 +12092,38 @@ define void @flat_atomic_fadd_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB151_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
-; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s6
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s7
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB151_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -12225,38 +12225,38 @@ define void @flat_atomic_fsub_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB153_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
-; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s6
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s7
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB153_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -12397,38 +12397,38 @@ define void @flat_atomic_fmax_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB155_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s6
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s7
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB155_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -12569,38 +12569,38 @@ define void @flat_atomic_fmin_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB157_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s6
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s7
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB157_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -12741,43 +12741,43 @@ define void @flat_atomic_fmaximum_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB159_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_max_f32_e32 v7, v2, v4
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v2, v4
-; GFX90A-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s6
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_max_f32_e32 v8, v7, v6
+; GFX90A-NEXT: v_max_f32_e32 v8, v2, v4
+; GFX90A-NEXT: v_max_f32_e32 v9, v7, v6
; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v8, vcc
-; GFX90A-NEXT: v_bfe_u32 v8, v7, 16, 1
-; GFX90A-NEXT: v_add3_u32 v8, v8, v7, s6
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v7
+; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX90A-NEXT: v_add3_u32 v8, v8, v2, s8
+; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s7
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v8, v9, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s9
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB159_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -12923,43 +12923,43 @@ define void @flat_atomic_fminimum_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB161_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_min_f32_e32 v7, v2, v4
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v2, v4
-; GFX90A-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s6
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_min_f32_e32 v8, v7, v6
+; GFX90A-NEXT: v_min_f32_e32 v8, v2, v4
+; GFX90A-NEXT: v_min_f32_e32 v9, v7, v6
; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v8, vcc
-; GFX90A-NEXT: v_bfe_u32 v8, v7, 16, 1
-; GFX90A-NEXT: v_add3_u32 v8, v8, v7, s6
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v7
+; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX90A-NEXT: v_add3_u32 v8, v8, v2, s8
+; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s7
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v8, v9, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s9
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB161_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -13490,27 +13490,28 @@ define void @flat_atomic_nand_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
+; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB172_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_and_b32_e32 v2, v3, v4
-; GFX90A-NEXT: v_not_b32_e32 v2, v2
-; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
+; GFX90A-NEXT: v_and_b32_e32 v0, v1, v4
+; GFX90A-NEXT: v_not_b32_e32 v0, v0
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB172_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; use v2
+; GFX90A-NEXT: ; use v0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -13518,8 +13519,9 @@ define void @flat_atomic_nand_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
@@ -13527,18 +13529,18 @@ define void @flat_atomic_nand_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: .LBB172_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_bitop3_b32 v2, v3, v4, v3 bitop3:0x3f
-; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_bitop3_b32 v0, v1, v4, v1 bitop3:0x3f
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB172_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v2
+; GFX950-NEXT: ; use v0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 10
@@ -14257,28 +14259,29 @@ define void @flat_atomic_usub_cond_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
+; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB190_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_sub_u32_e32 v2, v3, v4
-; GFX90A-NEXT: v_cmp_ge_u32_e32 vcc, v3, v4
-; GFX90A-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
+; GFX90A-NEXT: v_sub_u32_e32 v0, v1, v4
+; GFX90A-NEXT: v_cmp_ge_u32_e32 vcc, v1, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB190_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; use v2
+; GFX90A-NEXT: ; use v0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -14286,29 +14289,30 @@ define void @flat_atomic_usub_cond_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: .LBB190_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_sub_u32_e32 v2, v3, v4
-; GFX950-NEXT: v_cmp_ge_u32_e32 vcc, v3, v4
+; GFX950-NEXT: v_sub_u32_e32 v0, v1, v4
+; GFX950-NEXT: v_cmp_ge_u32_e32 vcc, v1, v4
; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
-; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB190_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v2
+; GFX950-NEXT: ; use v0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 10
@@ -14391,26 +14395,27 @@ define void @flat_atomic_usub_sat_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
+; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB192_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_sub_u32_e64 v2, v3, v4 clamp
-; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
+; GFX90A-NEXT: v_sub_u32_e64 v0, v1, v4 clamp
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB192_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; use v2
+; GFX90A-NEXT: ; use v0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -14418,8 +14423,9 @@ define void @flat_atomic_usub_sat_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
@@ -14427,18 +14433,18 @@ define void @flat_atomic_usub_sat_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: .LBB192_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_sub_u32_e64 v2, v3, v4 clamp
-; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_sub_u32_e64 v0, v1, v4 clamp
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB192_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v2
+; GFX950-NEXT: ; use v0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 10
@@ -17134,18 +17140,18 @@ define void @flat_atomic_udec_wrap_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v[0:1]
+; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_cbranch_scc0 .LBB218_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_atomic_dec_x2 v[2:3], v[2:3], v[0:1] sc0
+; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX950-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: s_branch .LBB218_3
; GFX950-NEXT: .LBB218_2:
; GFX950-NEXT: s_mov_b64 s[2:3], -1
-; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-NEXT: .LBB218_3: ; %Flow
; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: s_cselect_b32 s2, 1, 0
@@ -17154,21 +17160,21 @@ define void @flat_atomic_udec_wrap_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s2, s0, -1
-; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s2
+; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s2
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_subrev_co_u32_e32 v4, vcc, 1, v2
-; GFX950-NEXT: v_cmp_gt_u64_e64 s[0:1], v[2:3], v[0:1]
+; GFX950-NEXT: v_subrev_co_u32_e32 v4, vcc, 1, v0
+; GFX950-NEXT: v_cmp_gt_u64_e64 s[0:1], v[0:1], v[2:3]
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_subbrev_co_u32_e32 v5, vcc, 0, v3, vcc
+; GFX950-NEXT: v_subbrev_co_u32_e32 v5, vcc, 0, v1, vcc
; GFX950-NEXT: s_or_b64 vcc, vcc, s[0:1]
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX950-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s2
+; GFX950-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
+; GFX950-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
+; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s2
; GFX950-NEXT: .LBB218_5: ; %atomicrmw.phi
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v[2:3]
+; GFX950-NEXT: ; use v[0:1]
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 10
@@ -17959,26 +17965,27 @@ define void @flat_atomic_fsub_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
+; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB226_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_sub_f32_e32 v2, v3, v4
-; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
+; GFX90A-NEXT: v_sub_f32_e32 v0, v1, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB226_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; use v2
+; GFX90A-NEXT: ; use v0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -17986,26 +17993,27 @@ define void @flat_atomic_fsub_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: .LBB226_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_sub_f32_e32 v2, v3, v4
-; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_sub_f32_e32 v0, v1, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB226_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v2
+; GFX950-NEXT: ; use v0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 10
@@ -18398,8 +18406,9 @@ define void @flat_atomic_fmaximum_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
@@ -18407,18 +18416,18 @@ define void @flat_atomic_fmaximum_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: .LBB232_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_maximum3_f32 v2, v3, v4, v4
-; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_maximum3_f32 v0, v1, v4, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB232_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v2
+; GFX950-NEXT: ; use v0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 10
@@ -18535,8 +18544,9 @@ define void @flat_atomic_fminimum_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
@@ -18544,18 +18554,18 @@ define void @flat_atomic_fminimum_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: .LBB234_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_minimum3_f32 v2, v3, v4, v4
-; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_minimum3_f32 v0, v1, v4, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB234_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v2
+; GFX950-NEXT: ; use v0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 10
@@ -19254,18 +19264,18 @@ define void @flat_atomic_fmax_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v[0:1]
+; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_cbranch_scc0 .LBB240_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_atomic_max_f64 v[2:3], v[2:3], v[0:1] sc0
+; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX950-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: s_branch .LBB240_3
; GFX950-NEXT: .LBB240_2:
; GFX950-NEXT: s_mov_b64 s[2:3], -1
-; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-NEXT: .LBB240_3: ; %Flow
; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: s_cselect_b32 s2, 1, 0
@@ -19274,16 +19284,16 @@ define void @flat_atomic_fmax_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
-; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s0
-; GFX950-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s0
+; GFX950-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX950-NEXT: v_max_f64 v[0:1], v[4:5], v[0:1]
-; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s0
+; GFX950-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
+; GFX950-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]
+; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s0
; GFX950-NEXT: .LBB240_5: ; %atomicrmw.end
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v[2:3]
+; GFX950-NEXT: ; use v[0:1]
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 10
@@ -19460,18 +19470,18 @@ define void @flat_atomic_fmin_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v[0:1]
+; GFX950-NEXT: ; def v[2:3]
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_cbranch_scc0 .LBB242_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_atomic_min_f64 v[2:3], v[2:3], v[0:1] sc0
+; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; GFX950-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: s_branch .LBB242_3
; GFX950-NEXT: .LBB242_2:
; GFX950-NEXT: s_mov_b64 s[2:3], -1
-; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX950-NEXT: .LBB242_3: ; %Flow
; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: s_cselect_b32 s2, 1, 0
@@ -19480,16 +19490,16 @@ define void @flat_atomic_fmin_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
-; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s0
-; GFX950-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
+; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s0
+; GFX950-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX950-NEXT: v_min_f64 v[0:1], v[4:5], v[0:1]
-; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s0
+; GFX950-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
+; GFX950-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]
+; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s0
; GFX950-NEXT: .LBB242_5: ; %atomicrmw.end
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v[2:3]
+; GFX950-NEXT: ; use v[0:1]
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 10
@@ -20085,26 +20095,27 @@ define void @flat_atomic_fadd_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
+; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB248_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_add_f16 v2, v3, v4
-; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
+; GFX90A-NEXT: v_pk_add_f16 v0, v1, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB248_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; use v2
+; GFX90A-NEXT: ; use v0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -20202,26 +20213,27 @@ define void @flat_atomic_fsub_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
+; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB250_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_add_f16 v2, v3, v4 neg_lo:[0,1] neg_hi:[0,1]
-; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
+; GFX90A-NEXT: v_pk_add_f16 v0, v1, v4 neg_lo:[0,1] neg_hi:[0,1]
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB250_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; use v2
+; GFX90A-NEXT: ; use v0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -20229,8 +20241,9 @@ define void @flat_atomic_fsub_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
@@ -20238,18 +20251,18 @@ define void @flat_atomic_fsub_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: .LBB250_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_add_f16 v2, v3, v4 neg_lo:[0,1] neg_hi:[0,1]
-; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_pk_add_f16 v0, v1, v4 neg_lo:[0,1] neg_hi:[0,1]
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB250_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v2
+; GFX950-NEXT: ; use v0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 10
@@ -20553,28 +20566,28 @@ define void @flat_atomic_fmaximum_v2f16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
-; GFX90A-NEXT: s_mov_b32 s6, 0x5040100
+; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB255_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v0, v1, v4
-; GFX90A-NEXT: v_cmp_o_f16_e32 vcc, v1, v4
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v0, vcc
; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v1, v4 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v1, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v0, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_sdwa v0, v5, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_perm_b32 v0, v0, v6, s6
+; GFX90A-NEXT: v_perm_b32 v0, v0, v6, s8
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB255_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -20658,8 +20671,9 @@ define void @flat_atomic_fmaximum_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
@@ -20667,18 +20681,18 @@ define void @flat_atomic_fmaximum_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: .LBB256_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_maximum3_f16 v2, v3, v4, v4
-; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_pk_maximum3_f16 v0, v1, v4, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB256_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v2
+; GFX950-NEXT: ; use v0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 10
@@ -20698,28 +20712,28 @@ define void @flat_atomic_fminimum_v2f16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
-; GFX90A-NEXT: s_mov_b32 s6, 0x5040100
+; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB257_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_min_f16 v0, v1, v4
-; GFX90A-NEXT: v_cmp_o_f16_e32 vcc, v1, v4
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v0, vcc
; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v1, v4 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v1, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v0, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_sdwa v0, v5, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_perm_b32 v0, v0, v6, s6
+; GFX90A-NEXT: v_perm_b32 v0, v0, v6, s8
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB257_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -20803,8 +20817,9 @@ define void @flat_atomic_fminimum_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
@@ -20812,18 +20827,18 @@ define void @flat_atomic_fminimum_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: .LBB258_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_minimum3_f16 v2, v3, v4, v4
-; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_pk_minimum3_f16 v0, v1, v4, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB258_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v2
+; GFX950-NEXT: ; use v0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 10
@@ -20847,39 +20862,39 @@ define void @flat_atomic_fadd_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB259_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_add_f32_e32 v0, v0, v4
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s6
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v4
; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s7
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB259_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -20988,39 +21003,39 @@ define void @flat_atomic_fsub_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB261_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_sub_f32_e32 v0, v0, v4
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s6
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX90A-NEXT: v_sub_f32_e32 v0, v0, v4
; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s7
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB261_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -21168,39 +21183,39 @@ define void @flat_atomic_fmax_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB263_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_max_f32_e32 v0, v0, v4
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s6
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX90A-NEXT: v_max_f32_e32 v0, v0, v4
; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s7
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB263_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -21348,39 +21363,39 @@ define void @flat_atomic_fmin_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB265_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_min_f32_e32 v0, v0, v4
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s6
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX90A-NEXT: v_min_f32_e32 v0, v0, v4
; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s7
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB265_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -21528,44 +21543,44 @@ define void @flat_atomic_fmaximum_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB267_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_max_f32_e32 v7, v0, v4
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v0, v4
-; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v7, vcc
-; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s6
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc
; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_max_f32_e32 v8, v7, v6
+; GFX90A-NEXT: v_max_f32_e32 v8, v0, v4
+; GFX90A-NEXT: v_max_f32_e32 v9, v7, v6
; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v8, vcc
-; GFX90A-NEXT: v_bfe_u32 v8, v7, 16, 1
-; GFX90A-NEXT: v_add3_u32 v8, v8, v7, s6
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v7
+; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v0, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX90A-NEXT: v_add3_u32 v8, v8, v0, s8
+; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v7, v0, s7
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v8, v9, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v7, v0, s9
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB267_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -21718,44 +21733,44 @@ define void @flat_atomic_fminimum_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB269_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_min_f32_e32 v7, v0, v4
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v0, v4
-; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v7, vcc
-; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s6
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc
; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_min_f32_e32 v8, v7, v6
+; GFX90A-NEXT: v_min_f32_e32 v8, v0, v4
+; GFX90A-NEXT: v_min_f32_e32 v9, v7, v6
; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v8, vcc
-; GFX90A-NEXT: v_bfe_u32 v8, v7, 16, 1
-; GFX90A-NEXT: v_add3_u32 v8, v8, v7, s6
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v7
+; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v0, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX90A-NEXT: v_add3_u32 v8, v8, v0, s8
+; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v7, v0, s7
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v8, v9, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v7, v0, s9
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB269_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
diff --git a/llvm/test/CodeGen/AMDGPU/a-v-global-atomicrmw.ll b/llvm/test/CodeGen/AMDGPU/a-v-global-atomicrmw.ll
index 304a8501b904d9..4983695ac24ee9 100644
--- a/llvm/test/CodeGen/AMDGPU/a-v-global-atomicrmw.ll
+++ b/llvm/test/CodeGen/AMDGPU/a-v-global-atomicrmw.ll
@@ -7884,27 +7884,27 @@ define void @global_atomic_fmaximum_v2f16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
-; GFX90A-NEXT: s_mov_b32 s6, 0x5040100
+; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: .LBB147_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v2, v3, v4
-; GFX90A-NEXT: v_cmp_o_f16_e32 vcc, v3, v4
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v2, vcc
; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v3, v4 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v3, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v2, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_sdwa v2, v5, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s6
+; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s8
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB147_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -8022,27 +8022,27 @@ define void @global_atomic_fminimum_v2f16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
-; GFX90A-NEXT: s_mov_b32 s6, 0x5040100
+; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: .LBB149_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_min_f16 v2, v3, v4
-; GFX90A-NEXT: v_cmp_o_f16_e32 vcc, v3, v4
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v2, vcc
; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v3, v4 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v3, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v2, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_sdwa v2, v5, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s6
+; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s8
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB149_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -8164,38 +8164,38 @@ define void @global_atomic_fadd_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB151_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
-; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s6
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s7
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB151_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -8297,38 +8297,38 @@ define void @global_atomic_fsub_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB153_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
-; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s6
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s7
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB153_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -8469,38 +8469,38 @@ define void @global_atomic_fmax_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB155_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s6
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s7
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB155_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -8641,38 +8641,38 @@ define void @global_atomic_fmin_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB157_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s6
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s7
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB157_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -8813,43 +8813,43 @@ define void @global_atomic_fmaximum_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB159_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_max_f32_e32 v7, v2, v4
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v2, v4
-; GFX90A-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s6
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_max_f32_e32 v8, v7, v6
+; GFX90A-NEXT: v_max_f32_e32 v8, v2, v4
+; GFX90A-NEXT: v_max_f32_e32 v9, v7, v6
; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v8, vcc
-; GFX90A-NEXT: v_bfe_u32 v8, v7, 16, 1
-; GFX90A-NEXT: v_add3_u32 v8, v8, v7, s6
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v7
+; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX90A-NEXT: v_add3_u32 v8, v8, v2, s8
+; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s7
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v8, v9, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s9
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB159_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -8995,43 +8995,43 @@ define void @global_atomic_fminimum_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB161_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_min_f32_e32 v7, v2, v4
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v2, v4
-; GFX90A-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s6
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_min_f32_e32 v8, v7, v6
+; GFX90A-NEXT: v_min_f32_e32 v8, v2, v4
+; GFX90A-NEXT: v_min_f32_e32 v9, v7, v6
; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v8, vcc
-; GFX90A-NEXT: v_bfe_u32 v8, v7, 16, 1
-; GFX90A-NEXT: v_add3_u32 v8, v8, v7, s6
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v7
+; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX90A-NEXT: v_add3_u32 v8, v8, v2, s8
+; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s7
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v8, v9, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s9
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB161_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -13763,27 +13763,27 @@ define void @global_atomic_fmaximum_v2f16_saddr_ret_a_a(ptr addrspace(1) inreg %
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v3, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_mov_b32_e32 v4, 0x7e00
-; GFX90A-NEXT: s_mov_b32 s6, 0x5040100
+; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: .LBB255_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v0, v1, v3
-; GFX90A-NEXT: v_cmp_o_f16_e32 vcc, v1, v3
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v4, v0, vcc
; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v1, v3 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v1, v3
+; GFX90A-NEXT: v_cndmask_b32_e64 v5, v4, v0, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_sdwa v0, v4, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_perm_b32 v0, v0, v5, s6
+; GFX90A-NEXT: v_perm_b32 v0, v0, v5, s8
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB255_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -13905,27 +13905,27 @@ define void @global_atomic_fminimum_v2f16_saddr_ret_a_a(ptr addrspace(1) inreg %
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v3, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_mov_b32_e32 v4, 0x7e00
-; GFX90A-NEXT: s_mov_b32 s6, 0x5040100
+; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: .LBB257_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_min_f16 v0, v1, v3
-; GFX90A-NEXT: v_cmp_o_f16_e32 vcc, v1, v3
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v4, v0, vcc
; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v1, v3 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v1, v3
+; GFX90A-NEXT: v_cndmask_b32_e64 v5, v4, v0, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_sdwa v0, v4, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_perm_b32 v0, v0, v5, s6
+; GFX90A-NEXT: v_perm_b32 v0, v0, v5, s8
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB257_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -14051,38 +14051,38 @@ define void @global_atomic_fadd_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg %ptr
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB259_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_add_f32_e32 v0, v0, v3
-; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s6
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v3
; GFX90A-NEXT: v_add_f32_e32 v5, v5, v4
-; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s6
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v5
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s7
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB259_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -14188,38 +14188,38 @@ define void @global_atomic_fsub_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg %ptr
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB261_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_sub_f32_e32 v0, v0, v3
-; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s6
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX90A-NEXT: v_sub_f32_e32 v0, v0, v3
; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v4
-; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s6
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v5
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s7
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB261_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -14364,38 +14364,38 @@ define void @global_atomic_fmax_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg %ptr
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB263_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_max_f32_e32 v0, v0, v3
-; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s6
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX90A-NEXT: v_max_f32_e32 v0, v0, v3
; GFX90A-NEXT: v_max_f32_e32 v5, v5, v4
-; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s6
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v5
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s7
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB263_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -14540,38 +14540,38 @@ define void @global_atomic_fmin_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg %ptr
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB265_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_min_f32_e32 v0, v0, v3
-; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s6
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX90A-NEXT: v_min_f32_e32 v0, v0, v3
; GFX90A-NEXT: v_min_f32_e32 v5, v5, v4
-; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s6
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v5
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s7
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB265_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -14716,43 +14716,43 @@ define void @global_atomic_fmaximum_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v4, 0x7fc00000
-; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB267_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_max_f32_e32 v6, v0, v3
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v0, v3
-; GFX90A-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s6
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX90A-NEXT: v_max_f32_e32 v7, v6, v5
+; GFX90A-NEXT: v_max_f32_e32 v7, v0, v3
+; GFX90A-NEXT: v_max_f32_e32 v8, v6, v5
; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v6, v5
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v4, v7, vcc
-; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v0, v3
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v4, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v4, v8, vcc
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s7
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB267_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -14902,43 +14902,43 @@ define void @global_atomic_fminimum_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v4, 0x7fc00000
-; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB269_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_min_f32_e32 v6, v0, v3
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v0, v3
-; GFX90A-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s6
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX90A-NEXT: v_min_f32_e32 v7, v6, v5
+; GFX90A-NEXT: v_min_f32_e32 v7, v0, v3
+; GFX90A-NEXT: v_min_f32_e32 v8, v6, v5
; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v6, v5
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v4, v7, vcc
-; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v0, v3
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v4, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v4, v8, vcc
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s7
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB269_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
diff --git a/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll b/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll
index 431416e6f917a8..59f7e5c2b43f7c 100644
--- a/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll
@@ -775,7 +775,7 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX90A-NEXT: s_mov_b64 s[4:5], -1
; GFX90A-NEXT: s_xor_b64 s[6:7], s[6:7], s[4:5]
; GFX90A-NEXT: s_and_b64 vcc, exec, s[6:7]
-; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX90A-NEXT: s_cbranch_vccnz .LBB5_3
; GFX90A-NEXT: .LBB5_1: ; %Flow4
; GFX90A-NEXT: s_mov_b64 s[6:7], -1
@@ -783,7 +783,7 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX90A-NEXT: s_and_b64 vcc, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_vccnz .LBB5_10
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.shared
-; GFX90A-NEXT: ds_add_rtn_f64 v[4:5], v0, v[0:1]
+; GFX90A-NEXT: ds_add_rtn_f64 v[2:3], v0, v[0:1]
; GFX90A-NEXT: s_branch .LBB5_10
; GFX90A-NEXT: .LBB5_3: ; %atomicrmw.check.private
; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base
@@ -795,16 +795,16 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX90A-NEXT: s_mov_b64 s[4:5], -1
; GFX90A-NEXT: s_xor_b64 s[6:7], s[6:7], s[4:5]
; GFX90A-NEXT: s_and_b64 vcc, exec, s[6:7]
-; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX90A-NEXT: s_cbranch_vccnz .LBB5_5
; GFX90A-NEXT: s_branch .LBB5_6
; GFX90A-NEXT: .LBB5_4: ; %atomicrmw.private
-; GFX90A-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
+; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
-; GFX90A-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1]
-; GFX90A-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
-; GFX90A-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_add_f64 v[4:5], v[2:3], v[0:1]
+; GFX90A-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen
+; GFX90A-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen
; GFX90A-NEXT: s_branch .LBB5_9
; GFX90A-NEXT: .LBB5_5: ; %atomicrmw.global
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
@@ -829,8 +829,8 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX90A-NEXT: global_atomic_cmpswap_x2 v[2:3], v6, v[2:5], s[6:7] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u64_e64 s[6:7], v[2:3], v[4:5]
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
; GFX90A-NEXT: s_or_b64 s[4:5], s[6:7], s[4:5]
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB5_7
; GFX90A-NEXT: ; %bb.8: ; %atomicrmw.end1
@@ -844,9 +844,9 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX90A-NEXT: ; %bb.11: ; %atomicrmw.end
; GFX90A-NEXT: s_mov_b32 s4, 32
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b64 v[2:3], s4, v[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v4
-; GFX90A-NEXT: v_mov_b32_e32 v1, v2
+; GFX90A-NEXT: v_lshrrev_b64 v[4:5], s4, v[2:3]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -862,7 +862,7 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX942-NEXT: s_mov_b64 s[0:1], -1
; GFX942-NEXT: s_xor_b64 s[2:3], s[2:3], s[0:1]
; GFX942-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX942-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX942-NEXT: s_cbranch_vccnz .LBB5_3
; GFX942-NEXT: .LBB5_1: ; %Flow4
; GFX942-NEXT: s_mov_b64 s[2:3], -1
@@ -870,7 +870,7 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX942-NEXT: s_and_b64 vcc, exec, s[0:1]
; GFX942-NEXT: s_cbranch_vccnz .LBB5_10
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.shared
-; GFX942-NEXT: ds_add_rtn_f64 v[4:5], v0, v[0:1]
+; GFX942-NEXT: ds_add_rtn_f64 v[2:3], v0, v[0:1]
; GFX942-NEXT: s_branch .LBB5_10
; GFX942-NEXT: .LBB5_3: ; %atomicrmw.check.private
; GFX942-NEXT: s_mov_b64 s[0:1], src_private_base
@@ -882,14 +882,14 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX942-NEXT: s_mov_b64 s[0:1], -1
; GFX942-NEXT: s_xor_b64 s[2:3], s[2:3], s[0:1]
; GFX942-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX942-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX942-NEXT: s_cbranch_vccnz .LBB5_5
; GFX942-NEXT: s_branch .LBB5_6
; GFX942-NEXT: .LBB5_4: ; %atomicrmw.private
-; GFX942-NEXT: scratch_load_dwordx2 v[4:5], off, s0
+; GFX942-NEXT: scratch_load_dwordx2 v[2:3], off, s0
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1]
-; GFX942-NEXT: scratch_store_dwordx2 off, v[2:3], s0
+; GFX942-NEXT: v_add_f64 v[4:5], v[2:3], v[0:1]
+; GFX942-NEXT: scratch_store_dwordx2 off, v[4:5], s0
; GFX942-NEXT: s_branch .LBB5_9
; GFX942-NEXT: .LBB5_5: ; %atomicrmw.global
; GFX942-NEXT: v_mov_b32_e32 v2, 0
@@ -914,8 +914,8 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX942-NEXT: global_atomic_cmpswap_x2 v[2:3], v6, v[2:5], s[2:3] sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_cmp_eq_u64_e64 s[2:3], v[2:3], v[4:5]
-; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
; GFX942-NEXT: s_or_b64 s[0:1], s[2:3], s[0:1]
+; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB5_7
; GFX942-NEXT: ; %bb.8: ; %atomicrmw.end1
@@ -929,9 +929,9 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX942-NEXT: ; %bb.11: ; %atomicrmw.end
; GFX942-NEXT: s_mov_b32 s0, 32
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshrrev_b64 v[2:3], s0, v[4:5]
-; GFX942-NEXT: v_mov_b32_e32 v0, v4
-; GFX942-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-NEXT: v_lshrrev_b64 v[4:5], s0, v[2:3]
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll b/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
index 1d4ada34650c3a..fbcba25ba207cf 100644
--- a/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
@@ -7,16 +7,14 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: s_load_dword s0, s[4:5], 0x8
; CHECK-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x0
; CHECK-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x10
-; CHECK-NEXT: v_mov_b32_e32 v0, 0x9037ab78
-; CHECK-NEXT: v_mov_b32_e32 v1, 0x3e21eeb6
+; CHECK-NEXT: v_mov_b32_e32 v30, 0x9037ab78
+; CHECK-NEXT: v_mov_b32_e32 v31, 0x3e21eeb6
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_bitcmp1_b32 s0, 0
; CHECK-NEXT: s_cselect_b64 s[14:15], -1, 0
; CHECK-NEXT: s_xor_b64 s[16:17], s[14:15], -1
; CHECK-NEXT: s_bitcmp1_b32 s0, 8
-; CHECK-NEXT: v_accvgpr_write_b32 a4, v1 ; Reload Reuse
; CHECK-NEXT: s_cselect_b64 s[18:19], -1, 0
-; CHECK-NEXT: v_accvgpr_write_b32 a5, v0 ; Reload Reuse
; CHECK-NEXT: s_xor_b64 s[20:21], s[18:19], -1
; CHECK-NEXT: v_mov_b32_e32 v2, 0xa17f65f6
; CHECK-NEXT: v_mov_b32_e32 v3, 0xbe927e4f
@@ -34,14 +32,14 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: v_mov_b32_e32 v14, 0x8427b883
; CHECK-NEXT: v_mov_b32_e32 v15, 0x3fae1bb4
; CHECK-NEXT: s_and_b64 s[0:1], exec, s[14:15]
-; CHECK-NEXT: v_mov_b32_e32 v20, 0x57b87036
-; CHECK-NEXT: v_mov_b32_e32 v21, 0x3fb3b136
+; CHECK-NEXT: v_mov_b32_e32 v16, 0x57b87036
+; CHECK-NEXT: v_mov_b32_e32 v17, 0x3fb3b136
; CHECK-NEXT: s_and_b64 s[2:3], exec, s[18:19]
; CHECK-NEXT: v_mov_b32_e32 v18, 0x55555523
; CHECK-NEXT: v_mov_b32_e32 v19, 0xbfd55555
; CHECK-NEXT: s_and_b64 s[4:5], exec, s[16:17]
; CHECK-NEXT: v_mov_b32_e32 v0, 0
-; CHECK-NEXT: v_mov_b64_e32 v[30:31], 0
+; CHECK-NEXT: v_mov_b64_e32 v[20:21], 0
; CHECK-NEXT: ; implicit-def: $agpr0_agpr1
; CHECK-NEXT: ; implicit-def: $vgpr22_vgpr23
; CHECK-NEXT: s_branch .LBB0_2
@@ -66,11 +64,8 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: v_mov_b64_e32 v[24:25], s[10:11]
; CHECK-NEXT: flat_load_dwordx2 v[24:25], v[24:25]
-; CHECK-NEXT: v_accvgpr_read_b32 v27, a4 ; Reload Reuse
-; CHECK-NEXT: v_accvgpr_read_b32 v26, a5 ; Reload Reuse
+; CHECK-NEXT: v_mov_b64_e32 v[26:27], v[30:31]
; CHECK-NEXT: v_mov_b64_e32 v[28:29], v[2:3]
-; CHECK-NEXT: v_mov_b64_e32 v[16:17], v[20:21]
-; CHECK-NEXT: v_mov_b64_e32 v[30:31], 0
; CHECK-NEXT: v_accvgpr_write_b32 a2, 0
; CHECK-NEXT: v_accvgpr_write_b32 a3, 0
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -88,9 +83,10 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: v_fmac_f64_e32 v[26:27], 0, v[28:29]
; CHECK-NEXT: v_mov_b64_e32 v[28:29], v[14:15]
; CHECK-NEXT: v_fmac_f64_e32 v[28:29], 0, v[26:27]
-; CHECK-NEXT: v_fmac_f64_e32 v[16:17], 0, v[28:29]
-; CHECK-NEXT: v_mov_b64_e32 v[26:27], v[18:19]
-; CHECK-NEXT: v_fmac_f64_e32 v[26:27], 0, v[16:17]
+; CHECK-NEXT: v_mov_b64_e32 v[26:27], v[16:17]
+; CHECK-NEXT: v_fmac_f64_e32 v[26:27], 0, v[28:29]
+; CHECK-NEXT: v_mov_b64_e32 v[28:29], v[18:19]
+; CHECK-NEXT: v_fmac_f64_e32 v[28:29], 0, v[26:27]
; CHECK-NEXT: s_branch .LBB0_6
; CHECK-NEXT: .LBB0_5: ; %Flow
; CHECK-NEXT: ; in Loop: Header=BB0_6 Depth=2
@@ -99,8 +95,8 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: .LBB0_6: ; %.preheader1855.i.i.i3329
; CHECK-NEXT: ; Parent Loop BB0_2 Depth=1
; CHECK-NEXT: ; => This Inner Loop Header: Depth=2
-; CHECK-NEXT: v_accvgpr_read_b32 v29, a3
-; CHECK-NEXT: v_accvgpr_read_b32 v28, a2
+; CHECK-NEXT: v_accvgpr_read_b32 v27, a3
+; CHECK-NEXT: v_accvgpr_read_b32 v26, a2
; CHECK-NEXT: s_mov_b64 s[24:25], -1
; CHECK-NEXT: s_mov_b64 s[6:7], -1
; CHECK-NEXT: s_mov_b64 vcc, s[2:3]
@@ -116,8 +112,8 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: ; %bb.8: ; %.preheader1856.preheader.i.i.i3325
; CHECK-NEXT: ; in Loop: Header=BB0_6 Depth=2
; CHECK-NEXT: s_mov_b64 s[24:25], 0
-; CHECK-NEXT: v_accvgpr_write_b32 a2, v26
-; CHECK-NEXT: v_accvgpr_write_b32 a3, v27
+; CHECK-NEXT: v_accvgpr_write_b32 a2, v28
+; CHECK-NEXT: v_accvgpr_write_b32 a3, v29
; CHECK-NEXT: s_mov_b64 s[6:7], 0
; CHECK-NEXT: s_branch .LBB0_5
; CHECK-NEXT: .LBB0_9: ; in Loop: Header=BB0_2 Depth=1
@@ -137,17 +133,16 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: s_cbranch_vccz .LBB0_13
; CHECK-NEXT: ; %bb.12: ; %._crit_edge2105.i.i.i2330.loopexit
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: v_cmp_nlg_f64_e64 s[6:7], 0, v[28:29]
+; CHECK-NEXT: v_cmp_nlg_f64_e64 s[6:7], 0, v[26:27]
; CHECK-NEXT: v_cndmask_b32_e64 v23, v23, 0, s[14:15]
; CHECK-NEXT: v_cndmask_b32_e64 v22, v22, 0, s[14:15]
-; CHECK-NEXT: v_cndmask_b32_e64 v16, 0, 1, s[6:7]
-; CHECK-NEXT: v_mov_b32_e32 v17, v16
+; CHECK-NEXT: v_cndmask_b32_e64 v26, 0, 1, s[6:7]
+; CHECK-NEXT: v_mov_b32_e32 v27, v26
; CHECK-NEXT: s_and_b64 s[6:7], exec, s[14:15]
-; CHECK-NEXT: global_store_dwordx2 v0, v[16:17], s[8:9]
+; CHECK-NEXT: global_store_dwordx2 v0, v[26:27], s[8:9]
; CHECK-NEXT: s_cselect_b32 s23, s23, 0
; CHECK-NEXT: s_cselect_b32 s22, s22, 0
; CHECK-NEXT: s_mov_b64 s[6:7], -1
-; CHECK-NEXT: v_mov_b64_e32 v[30:31], 0
; CHECK-NEXT: s_branch .LBB0_14
; CHECK-NEXT: .LBB0_13: ; in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: s_mov_b64 s[6:7], 0
@@ -164,7 +159,7 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: ; %bb.16: ; %._crit_edge2105.i.i.i2330
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: s_mov_b64 s[24:25], 0
-; CHECK-NEXT: global_store_dwordx2 v0, v[30:31], s[8:9]
+; CHECK-NEXT: global_store_dwordx2 v0, v[20:21], s[8:9]
; CHECK-NEXT: s_branch .LBB0_1
; CHECK-NEXT: .LBB0_17: ; %DummyReturnBlock
; CHECK-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/copy-to-reg-frameindex.ll b/llvm/test/CodeGen/AMDGPU/copy-to-reg-frameindex.ll
index 04c40e7ccdaf6b..f7e9b8724b6821 100644
--- a/llvm/test/CodeGen/AMDGPU/copy-to-reg-frameindex.ll
+++ b/llvm/test/CodeGen/AMDGPU/copy-to-reg-frameindex.ll
@@ -43,11 +43,10 @@ define void @phi_with_alloca_and_divergent_copy_to_reg(ptr addrspace(5) %diverge
; CHECK-LABEL: phi_with_alloca_and_divergent_copy_to_reg:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: s_lshr_b32 s6, s32, 6
; CHECK-NEXT: v_mov_b32_e32 v7, v2
; CHECK-NEXT: v_mov_b32_e32 v6, v1
; CHECK-NEXT: s_mov_b64 s[4:5], 0
-; CHECK-NEXT: v_mov_b32_e32 v2, s6
+; CHECK-NEXT: v_lshrrev_b32_e64 v2, 6, s32
; CHECK-NEXT: .LBB1_1: ; %loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: v_mov_b32_e32 v1, v2
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
index bb2dc353ab9fa5..f40b209b77da7a 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
@@ -14644,20 +14644,19 @@ define void @flat_atomic_fmax_f32_saddr_nortn(ptr inreg %ptr, float %data) {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[2:3] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: v_max_f32_e32 v4, v0, v0
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: .LBB119_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX950-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX950-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB119_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14721,20 +14720,19 @@ define void @flat_atomic_fmin_f32_saddr_nortn(ptr inreg %ptr, float %data) {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[2:3] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: v_max_f32_e32 v4, v0, v0
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: .LBB121_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX950-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX950-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB121_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14896,22 +14894,21 @@ define void @flat_atomic_fmax_v2f16_saddr_nortn(ptr inreg %ptr, <2 x half> %data
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[2:3] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: v_pk_max_f16 v4, v0, v0
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB125_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX950-NEXT: v_pk_max_f16 v0, v1, v1
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_max_f16 v2, v2, v4
-; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_pk_max_f16 v0, v0, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB125_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -15009,22 +15006,21 @@ define void @flat_atomic_fmin_v2f16_saddr_nortn(ptr inreg %ptr, <2 x half> %data
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[2:3] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: v_pk_max_f16 v4, v0, v0
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB127_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX950-NEXT: v_pk_max_f16 v0, v1, v1
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_min_f16 v2, v2, v4
-; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_pk_min_f16 v0, v0, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v2
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB127_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/insert-waitcnts-merge.ll b/llvm/test/CodeGen/AMDGPU/insert-waitcnts-merge.ll
index 03d1abcb053719..fadb15dd2fb799 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-waitcnts-merge.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert-waitcnts-merge.ll
@@ -23,9 +23,9 @@ define amdgpu_kernel void @widget(ptr addrspace(1) %arg, i1 %arg1) #0 {
; CHECK-NEXT: S_WAITCNT .Lgkmcnt_0
; CHECK-NEXT: S_BITCMP1_B32 killed renamable $sgpr2, 0, implicit-def $scc
; CHECK-NEXT: renamable $sgpr4_sgpr5 = S_MOV_B64 0
+ ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
; CHECK-NEXT: renamable $sgpr2_sgpr3 = S_CSELECT_B64 -1, 0, implicit killed $scc
; CHECK-NEXT: $vgpr2_vgpr3 = V_PK_MOV_B32 8, 0, 8, 0, 0, 0, 0, 0, 0, implicit $exec
- ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; CHECK-NEXT: renamable $sgpr6_sgpr7 = IMPLICIT_DEF
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
index 571c8aa6631c75..e5e07ff1a94eb1 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
@@ -6430,19 +6430,19 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-LABEL: local_atomic_fadd_ret_v2bf16:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v4, v0
+; GFX90A-NEXT: ds_read_b32 v3, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v4
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX90A-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX90A-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX90A-NEXT: v_add_f32_e32 v5, v5, v4
; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6454,11 +6454,11 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6791,19 +6791,19 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-LABEL: local_atomic_fadd_ret_v2bf16__offset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v4, v0 offset:65532
+; GFX90A-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v4
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX90A-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX90A-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX90A-NEXT: v_add_f32_e32 v5, v5, v4
; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6815,11 +6815,11 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1 offset:65532
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB25_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
index 671a8c49025bba..b74f6b1c346012 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
@@ -6157,19 +6157,19 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX942-LABEL: local_atomic_fmax_ret_v2bf16:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: ds_read_b32 v4, v0
+; GFX942-NEXT: ds_read_b32 v3, v0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
-; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v4
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX942-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX942-NEXT: v_max_f32_e32 v5, v5, v3
+; GFX942-NEXT: v_max_f32_e32 v5, v5, v4
; GFX942-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6182,11 +6182,11 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX942-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[0:1]
; GFX942-NEXT: v_perm_b32 v1, v5, v1, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v3, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB24_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6329,19 +6329,19 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-LABEL: local_atomic_fmax_ret_v2bf16:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v4, v0
+; GFX90A-NEXT: ds_read_b32 v3, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v4
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX90A-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX90A-NEXT: v_max_f32_e32 v5, v5, v3
+; GFX90A-NEXT: v_max_f32_e32 v5, v5, v4
; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6353,11 +6353,11 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6643,19 +6643,19 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX942-LABEL: local_atomic_fmax_ret_v2bf16__offset:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: ds_read_b32 v4, v0 offset:65532
+; GFX942-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX942-NEXT: s_mov_b64 s[2:3], 0
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
-; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v4
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX942-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX942-NEXT: v_max_f32_e32 v5, v5, v3
+; GFX942-NEXT: v_max_f32_e32 v5, v5, v4
; GFX942-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6668,11 +6668,11 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX942-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[0:1]
; GFX942-NEXT: v_perm_b32 v1, v5, v1, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1 offset:65532
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v3, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB25_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6815,19 +6815,19 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-LABEL: local_atomic_fmax_ret_v2bf16__offset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v4, v0 offset:65532
+; GFX90A-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v4
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX90A-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX90A-NEXT: v_max_f32_e32 v5, v5, v3
+; GFX90A-NEXT: v_max_f32_e32 v5, v5, v4
; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6839,11 +6839,11 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1 offset:65532
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB25_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
index e9f671c44cffbe..7c68c45a70a038 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
@@ -6157,19 +6157,19 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX942-LABEL: local_atomic_fmin_ret_v2bf16:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: ds_read_b32 v4, v0
+; GFX942-NEXT: ds_read_b32 v3, v0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
-; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v4
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX942-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX942-NEXT: v_min_f32_e32 v5, v5, v3
+; GFX942-NEXT: v_min_f32_e32 v5, v5, v4
; GFX942-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6182,11 +6182,11 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX942-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[0:1]
; GFX942-NEXT: v_perm_b32 v1, v5, v1, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v3, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB24_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6329,19 +6329,19 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-LABEL: local_atomic_fmin_ret_v2bf16:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v4, v0
+; GFX90A-NEXT: ds_read_b32 v3, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v4
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX90A-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX90A-NEXT: v_min_f32_e32 v5, v5, v3
+; GFX90A-NEXT: v_min_f32_e32 v5, v5, v4
; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6353,11 +6353,11 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6643,19 +6643,19 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX942-LABEL: local_atomic_fmin_ret_v2bf16__offset:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: ds_read_b32 v4, v0 offset:65532
+; GFX942-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX942-NEXT: s_mov_b64 s[2:3], 0
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
-; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v4
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX942-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX942-NEXT: v_min_f32_e32 v5, v5, v3
+; GFX942-NEXT: v_min_f32_e32 v5, v5, v4
; GFX942-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6668,11 +6668,11 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX942-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[0:1]
; GFX942-NEXT: v_perm_b32 v1, v5, v1, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1 offset:65532
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v3, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB25_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6815,19 +6815,19 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-LABEL: local_atomic_fmin_ret_v2bf16__offset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v4, v0 offset:65532
+; GFX90A-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v4
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX90A-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX90A-NEXT: v_min_f32_e32 v5, v5, v3
+; GFX90A-NEXT: v_min_f32_e32 v5, v5, v4
; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6839,11 +6839,11 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1 offset:65532
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB25_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
index 38723293fabb3d..5c988fcda1a8e4 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
@@ -6971,19 +6971,19 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX942-LABEL: local_atomic_fsub_ret_v2bf16:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: ds_read_b32 v4, v0
+; GFX942-NEXT: ds_read_b32 v3, v0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
-; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v4
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX942-NEXT: v_sub_f32_e32 v1, v1, v2
-; GFX942-NEXT: v_sub_f32_e32 v5, v5, v3
+; GFX942-NEXT: v_sub_f32_e32 v5, v5, v4
; GFX942-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6996,11 +6996,11 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX942-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[0:1]
; GFX942-NEXT: v_perm_b32 v1, v5, v1, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v3, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB24_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7143,19 +7143,19 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-LABEL: local_atomic_fsub_ret_v2bf16:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v4, v0
+; GFX90A-NEXT: ds_read_b32 v3, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v4
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX90A-NEXT: v_sub_f32_e32 v1, v1, v2
-; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v3
+; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v4
; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -7167,11 +7167,11 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7457,19 +7457,19 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX942-LABEL: local_atomic_fsub_ret_v2bf16__offset:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: ds_read_b32 v4, v0 offset:65532
+; GFX942-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX942-NEXT: s_mov_b64 s[2:3], 0
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
-; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v4
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX942-NEXT: v_sub_f32_e32 v1, v1, v2
-; GFX942-NEXT: v_sub_f32_e32 v5, v5, v3
+; GFX942-NEXT: v_sub_f32_e32 v5, v5, v4
; GFX942-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -7482,11 +7482,11 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX942-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[0:1]
; GFX942-NEXT: v_perm_b32 v1, v5, v1, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1 offset:65532
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v4, v1
+; GFX942-NEXT: v_mov_b32_e32 v3, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB25_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7629,19 +7629,19 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-LABEL: local_atomic_fsub_ret_v2bf16__offset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v4, v0 offset:65532
+; GFX90A-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v4
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
; GFX90A-NEXT: v_sub_f32_e32 v1, v1, v2
-; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v3
+; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v4
; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -7653,11 +7653,11 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1 offset:65532
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v4, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB25_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/phi-av-pressure.ll b/llvm/test/CodeGen/AMDGPU/phi-av-pressure.ll
index 29f26bb00135b3..a4bf3705534bc4 100644
--- a/llvm/test/CodeGen/AMDGPU/phi-av-pressure.ll
+++ b/llvm/test/CodeGen/AMDGPU/phi-av-pressure.ll
@@ -36,9 +36,9 @@ define amdgpu_kernel void @main(i1 %arg, ptr %ptr, ptr addrspace(1) %ptr1, ptr a
; GFX950-NEXT: v_lshl_add_u64 v[10:11], v[0:1], 3, s[38:39]
; GFX950-NEXT: global_load_dwordx4 v[2:5], v[10:11], off
; GFX950-NEXT: global_load_dwordx4 v[6:9], v1, s[38:39]
+; GFX950-NEXT: v_mov_b32_e32 v0, 0
; GFX950-NEXT: v_mov_b64_e32 v[10:11], s[36:37]
; GFX950-NEXT: s_and_b64 vcc, exec, s[34:35]
-; GFX950-NEXT: v_mov_b32_e32 v0, v1
; GFX950-NEXT: v_mov_b32_e32 v12, v1
; GFX950-NEXT: v_mov_b32_e32 v14, v1
; GFX950-NEXT: v_mov_b32_e32 v15, v1
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-spill-cost-reset.ll b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-spill-cost-reset.ll
index 1b2b93f1e12d4c..d115f9bd028f27 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-spill-cost-reset.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-spill-cost-reset.ll
@@ -68,62 +68,86 @@ define amdgpu_kernel void @test_spill_cost_reset(
; CHECK-NEXT: liveins: $sgpr4_sgpr5
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
- ; CHECK-NEXT: early-clobber %1843:sgpr_256 = S_LOAD_DWORDX8_IMM_ec [[COPY]](p4), 16, 0 :: (dereferenceable invariant load (s256) from %ir.a0.kernarg.offset, align 16, addrspace 4)
- ; CHECK-NEXT: early-clobber %1844:sgpr_256 = S_LOAD_DWORDX8_IMM_ec [[COPY]](p4), 48, 0 :: (dereferenceable invariant load (s256) from %ir.b0.kernarg.offset, align 16, addrspace 4)
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; CHECK-NEXT: early-clobber %1891:sgpr_256 = S_LOAD_DWORDX8_IMM_ec [[COPY]](p4), 16, 0 :: (dereferenceable invariant load (s256) from %ir.a0.kernarg.offset, align 16, addrspace 4)
+ ; CHECK-NEXT: early-clobber %1892:sgpr_256 = S_LOAD_DWORDX8_IMM_ec [[COPY]](p4), 48, 0 :: (dereferenceable invariant load (s256) from %ir.b0.kernarg.offset, align 16, addrspace 4)
; CHECK-NEXT: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]](p4), 80, 0 :: (dereferenceable invariant load (s32) from %ir.n.kernarg.offset, align 16, addrspace 4)
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_128_align2 = COPY %1843.sub0_sub1_sub2_sub3
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_128_align2 = COPY %1844.sub0_sub1_sub2_sub3
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:av_128_align2 = COPY %1843.sub4_sub5_sub6_sub7
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:av_128_align2 = COPY %1844.sub4_sub5_sub6_sub7
- ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY5:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY5:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY5:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY5:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY6:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY6:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY6:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY6:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY7:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY7:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY7:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY7:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY8:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY8:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY8:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY8:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY9:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY9:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY9:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY9:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY10:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY10:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY10:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY10:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY11:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY11:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY11:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY11:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY12:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY12:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY12:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY12:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY13:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY13:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY13:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY13:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY14:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY14:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY14:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY14:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY15:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY15:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY15:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY15:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_1:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_2:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_3:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_4:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_5:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_5:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_5:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_5:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_6:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_6:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_6:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_6:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_7:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_7:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_7:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_7:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_8:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_8:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_8:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_8:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_9:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_9:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_9:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_9:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_10:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_10:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_10:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_10:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_11:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_11:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_11:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_11:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_12:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_12:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_12:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_12:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_128_align2 = COPY %1891.sub0_sub1_sub2_sub3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_128_align2 = COPY %1892.sub0_sub1_sub2_sub3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:av_128_align2 = COPY %1891.sub4_sub5_sub6_sub7
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:av_128_align2 = COPY %1892.sub4_sub5_sub6_sub7
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY5:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY6:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY7:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY8:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY9:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY10:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY11:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY12:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY13:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY14:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY15:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
; CHECK-NEXT: undef [[COPY16:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
; CHECK-NEXT: [[COPY16:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
; CHECK-NEXT: undef [[COPY17:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
@@ -356,105 +380,69 @@ define amdgpu_kernel void @test_spill_cost_reset(
; CHECK-NEXT: [[COPY130:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
; CHECK-NEXT: undef [[COPY131:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
; CHECK-NEXT: [[COPY131:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY132:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY132:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY133:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY133:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY134:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY134:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY135:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY135:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY136:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY136:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY137:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY137:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY138:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY138:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY139:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY139:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY140:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY140:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY141:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY141:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY142:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY142:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY143:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY143:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1.loop:
; CHECK-NEXT: successors: %bb.2(0x04000000), %bb.1(0x7c000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY15]], 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY143:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY143]], 8, [[COPY143]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY142:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY142]], 8, [[COPY142]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY141:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY141]], 8, [[COPY141]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY140:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY140]], 8, [[COPY140]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY139:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY139]], 8, [[COPY139]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY14]], 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY138:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY138]], 8, [[COPY138]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY137:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY137]], 8, [[COPY137]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY136:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY136]], 8, [[COPY136]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY13]], 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY135:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY135]], 8, [[COPY135]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY134:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY134]], 8, [[COPY134]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY133:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY133]], 8, [[COPY133]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY12]], 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY132:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY132]], 8, [[COPY132]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_12:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_12]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY131:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY131]], 8, [[COPY131]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY130:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY130]], 8, [[COPY130]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY11]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY129:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY129]], 8, [[COPY129]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY128:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY128]], 8, [[COPY128]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY127:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY127]], 8, [[COPY127]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY10]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_11:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_11]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY126:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY126]], 8, [[COPY126]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY125:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY125]], 8, [[COPY125]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY124:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY124]], 8, [[COPY124]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY9]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_10:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_10]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY123:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY123]], 8, [[COPY123]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY122:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY122]], 8, [[COPY122]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY121:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY121]], 8, [[COPY121]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY8]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_9:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_9]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY120:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY120]], 8, [[COPY120]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY119:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY119]], 8, [[COPY119]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY118:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY118]], 8, [[COPY118]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY7]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_8:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_8]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY117:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY117]], 8, [[COPY117]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY116:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY116]], 8, [[COPY116]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY115:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY115]], 8, [[COPY115]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY6]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_7:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_7]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY114:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY114]], 8, [[COPY114]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY113:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY113]], 8, [[COPY113]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY112:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY112]], 8, [[COPY112]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY5]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_6:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_6]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY111:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY111]], 8, [[COPY111]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY110:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY110]], 8, [[COPY110]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY109:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY109]], 8, [[COPY109]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_5:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_5]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY108:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY108]], 8, [[COPY108]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY107:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY107]], 8, [[COPY107]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY106:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY106]], 8, [[COPY106]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_4]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY105:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY105]], 8, [[COPY105]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY104:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY104]], 8, [[COPY104]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY103:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY103]], 8, [[COPY103]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY3]], [[COPY4]], [[COPY15]], 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_1:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_3]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY102:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY102]], 8, [[COPY102]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY101:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY101]], 8, [[COPY101]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY100:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY100]], 8, [[COPY100]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_2:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY3]], [[COPY4]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_2]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY99:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY99]], 8, [[COPY99]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY98:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY98]], 8, [[COPY98]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY97:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY97]], 8, [[COPY97]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_1]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY96:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY96]], 8, [[COPY96]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY95:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY95]], 8, [[COPY95]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY94:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY94]], 8, [[COPY94]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY93:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY93]], 8, [[COPY93]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY92:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY92]], 8, [[COPY92]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY91:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY91]], 8, [[COPY91]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY3]], [[COPY4]], [[V_MOV_B32_e32_12]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_1:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY90:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY90]], 8, [[COPY90]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY89:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY89]], 8, [[COPY89]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY88:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY88]], 8, [[COPY88]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_2:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY3]], [[COPY4]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY87:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY87]], 8, [[COPY87]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY86:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY86]], 8, [[COPY86]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY85:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY85]], 8, [[COPY85]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
@@ -526,16 +514,28 @@ define amdgpu_kernel void @test_spill_cost_reset(
; CHECK-NEXT: [[COPY19:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY19]], 8, [[COPY19]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY18:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY18]], 8, [[COPY18]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY17:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY17]], 8, [[COPY17]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY16:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY16]], 8, [[COPY16]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY15]], 8, [[COPY15]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY14]], 8, [[COPY14]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY13]], 8, [[COPY13]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY12]], 8, [[COPY12]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY11]], 8, [[COPY11]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY10]], 8, [[COPY10]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY9]], 8, [[COPY9]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY8]], 8, [[COPY8]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY7]], 8, [[COPY7]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY6]], 8, [[COPY6]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY5]], 8, [[COPY5]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_MOV_B32_]], 1, implicit-def dead $scc
; CHECK-NEXT: S_CMP_LT_I32 [[S_MOV_B32_]], [[S_LOAD_DWORD_IMM]], implicit-def $scc
- ; CHECK-NEXT: [[COPY16:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY16]], 8, [[COPY16]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_]], 8, [[V_MOV_B32_e32_]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: S_CBRANCH_SCC1 %bb.1, implicit killed $scc
; CHECK-NEXT: S_BRANCH %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2.epilogue:
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 0, 0 :: (dereferenceable invariant load (s64) from %ir.out.kernarg.offset550, align 16, addrspace 4)
- ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: GLOBAL_STORE_DWORD_SADDR [[V_MOV_B32_e32_1]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_2]].sub0, [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s32) into %ir.out.load, addrspace 1)
+ ; CHECK-NEXT: [[V_MOV_B32_e32_13:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD_SADDR [[V_MOV_B32_e32_13]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_2]].sub0, [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s32) into %ir.out.load, addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
ptr addrspace(1) %out,
<8 x half> %a0, <8 x half> %a1,
diff --git a/llvm/test/CodeGen/AMDGPU/si-fix-sgpr-copies-phi-av-classes.mir b/llvm/test/CodeGen/AMDGPU/si-fix-sgpr-copies-phi-av-classes.mir
deleted file mode 100644
index ee2974411e328f..00000000000000
--- a/llvm/test/CodeGen/AMDGPU/si-fix-sgpr-copies-phi-av-classes.mir
+++ /dev/null
@@ -1,386 +0,0 @@
-# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=amdgpu9.0a -run-pass=si-fix-sgpr-copies -verify-machineinstrs -o - %s | FileCheck %s
-
-# PHI operand legalization must not insert copies between vector register
-# classes that only differ in AV-ness when their common subclass is a VGPR
-# class (e.g. av_32 vs. vgpr_32). Copies from SGPRs, between VGPR and AGPR
-# classes, and between AV and AGPR classes are still required.
-
----
-name: phi_vgpr_av_operand
-tracksRegLiveness: true
-body: |
- ; CHECK-LABEL: name: phi_vgpr_av_operand
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; CHECK-NEXT: liveins: $vgpr0, $vgpr1, $sgpr0_sgpr1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:av_32 = COPY $vgpr0
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
- ; CHECK-NEXT: $vcc = COPY [[COPY2]]
- ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
- ; CHECK-NEXT: S_BRANCH %bb.1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[COPY1]], [[COPY1]], implicit $exec
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY]], %bb.0, [[V_ADD_U32_e32_]], %bb.1
- ; CHECK-NEXT: $vgpr0 = COPY [[PHI]]
- ; CHECK-NEXT: SI_RETURN implicit $vgpr0
- bb.0:
- successors: %bb.1, %bb.2
- liveins: $vgpr0, $vgpr1, $sgpr0_sgpr1
-
- %0:av_32 = COPY $vgpr0
- %1:vgpr_32 = COPY $vgpr1
- %2:sreg_64 = COPY $sgpr0_sgpr1
- $vcc = COPY %2
- S_CBRANCH_VCCNZ %bb.2, implicit $vcc
- S_BRANCH %bb.1
-
- bb.1:
- successors: %bb.2
- %3:vgpr_32 = V_ADD_U32_e32 %1, %1, implicit $exec
-
- bb.2:
- %4:vgpr_32 = PHI %0, %bb.0, %3, %bb.1
- $vgpr0 = COPY %4
- SI_RETURN implicit $vgpr0
-...
-
----
-name: phi_av_vgpr_operand
-tracksRegLiveness: true
-body: |
- ; CHECK-LABEL: name: phi_av_vgpr_operand
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; CHECK-NEXT: liveins: $vgpr0, $vgpr1, $sgpr0_sgpr1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_32 = COPY $vgpr1
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
- ; CHECK-NEXT: $vcc = COPY [[COPY2]]
- ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
- ; CHECK-NEXT: S_BRANCH %bb.1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:av_32 = COPY [[COPY1]]
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:av_32 = PHI [[COPY]], %bb.0, [[COPY3]], %bb.1
- ; CHECK-NEXT: $vgpr0 = COPY [[PHI]]
- ; CHECK-NEXT: SI_RETURN implicit $vgpr0
- bb.0:
- successors: %bb.1, %bb.2
- liveins: $vgpr0, $vgpr1, $sgpr0_sgpr1
-
- %0:vgpr_32 = COPY $vgpr0
- %1:av_32 = COPY $vgpr1
- %2:sreg_64 = COPY $sgpr0_sgpr1
- $vcc = COPY %2
- S_CBRANCH_VCCNZ %bb.2, implicit $vcc
- S_BRANCH %bb.1
-
- bb.1:
- successors: %bb.2
- %3:av_32 = COPY %1
-
- bb.2:
- %4:av_32 = PHI %0, %bb.0, %3, %bb.1
- $vgpr0 = COPY %4
- SI_RETURN implicit $vgpr0
-...
-
----
-name: phi_av64_vreg64_operand
-tracksRegLiveness: true
-body: |
- ; CHECK-LABEL: name: phi_av64_vreg64_operand
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $sgpr0_sgpr1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
- ; CHECK-NEXT: $vcc = COPY [[COPY2]]
- ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
- ; CHECK-NEXT: S_BRANCH %bb.1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:av_64_align2 = COPY [[COPY1]]
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:av_64_align2 = PHI [[COPY]], %bb.0, [[COPY3]], %bb.1
- ; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[PHI]]
- ; CHECK-NEXT: SI_RETURN implicit $vgpr0_vgpr1
- bb.0:
- successors: %bb.1, %bb.2
- liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $sgpr0_sgpr1
-
- %0:vreg_64_align2 = COPY $vgpr0_vgpr1
- %1:av_64_align2 = COPY $vgpr2_vgpr3
- %2:sreg_64 = COPY $sgpr0_sgpr1
- $vcc = COPY %2
- S_CBRANCH_VCCNZ %bb.2, implicit $vcc
- S_BRANCH %bb.1
-
- bb.1:
- successors: %bb.2
- %3:av_64_align2 = COPY %1
-
- bb.2:
- %4:av_64_align2 = PHI %0, %bb.0, %3, %bb.1
- $vgpr0_vgpr1 = COPY %4
- SI_RETURN implicit $vgpr0_vgpr1
-...
-
-# The SGPR operand still needs a copy to a vector register (see #168761).
----
-name: phi_av_sgpr_operand
-tracksRegLiveness: true
-body: |
- ; CHECK-LABEL: name: phi_av_sgpr_operand
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; CHECK-NEXT: liveins: $vgpr0, $sgpr0_sgpr1, $sgpr2
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_32 = COPY $vgpr0
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
- ; CHECK-NEXT: $vcc = COPY [[COPY2]]
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:av_32 = COPY [[COPY]], implicit $exec
- ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
- ; CHECK-NEXT: S_BRANCH %bb.1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:av_32 = COPY [[COPY1]]
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:av_32 = PHI [[COPY3]], %bb.0, [[COPY4]], %bb.1
- ; CHECK-NEXT: $vgpr0 = COPY [[PHI]]
- ; CHECK-NEXT: SI_RETURN implicit $vgpr0
- bb.0:
- successors: %bb.1, %bb.2
- liveins: $vgpr0, $sgpr0_sgpr1, $sgpr2
-
- %0:sreg_32 = COPY $sgpr2
- %1:av_32 = COPY $vgpr0
- %2:sreg_64 = COPY $sgpr0_sgpr1
- $vcc = COPY %2
- S_CBRANCH_VCCNZ %bb.2, implicit $vcc
- S_BRANCH %bb.1
-
- bb.1:
- successors: %bb.2
- %3:av_32 = COPY %1
-
- bb.2:
- %4:av_32 = PHI %0, %bb.0, %3, %bb.1
- $vgpr0 = COPY %4
- SI_RETURN implicit $vgpr0
-...
-
-# An AGPR operand of a VGPR PHI still needs a copy.
----
-name: phi_vgpr_agpr_operand
-tracksRegLiveness: true
-body: |
- ; CHECK-LABEL: name: phi_vgpr_agpr_operand
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; CHECK-NEXT: liveins: $vgpr0, $agpr0, $sgpr0_sgpr1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:agpr_32 = COPY $agpr0
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
- ; CHECK-NEXT: $vcc = COPY [[COPY2]]
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]], implicit $exec
- ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
- ; CHECK-NEXT: S_BRANCH %bb.1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[COPY1]], [[COPY1]], implicit $exec
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, [[V_ADD_U32_e32_]], %bb.1
- ; CHECK-NEXT: $vgpr0 = COPY [[PHI]]
- ; CHECK-NEXT: SI_RETURN implicit $vgpr0
- bb.0:
- successors: %bb.1, %bb.2
- liveins: $vgpr0, $agpr0, $sgpr0_sgpr1
-
- %0:agpr_32 = COPY $agpr0
- %1:vgpr_32 = COPY $vgpr0
- %2:sreg_64 = COPY $sgpr0_sgpr1
- $vcc = COPY %2
- S_CBRANCH_VCCNZ %bb.2, implicit $vcc
- S_BRANCH %bb.1
-
- bb.1:
- successors: %bb.2
- %3:vgpr_32 = V_ADD_U32_e32 %1, %1, implicit $exec
-
- bb.2:
- %4:vgpr_32 = PHI %0, %bb.0, %3, %bb.1
- $vgpr0 = COPY %4
- SI_RETURN implicit $vgpr0
-...
-
-# A subregister operand keeps the copy.
----
-name: phi_vgpr_av64_subreg_operand
-tracksRegLiveness: true
-body: |
- ; CHECK-LABEL: name: phi_vgpr_av64_subreg_operand
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0_sgpr1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
- ; CHECK-NEXT: $vcc = COPY [[COPY2]]
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1, implicit $exec
- ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
- ; CHECK-NEXT: S_BRANCH %bb.1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[COPY1]], [[COPY1]], implicit $exec
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, [[V_ADD_U32_e32_]], %bb.1
- ; CHECK-NEXT: $vgpr0 = COPY [[PHI]]
- ; CHECK-NEXT: SI_RETURN implicit $vgpr0
- bb.0:
- successors: %bb.1, %bb.2
- liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0_sgpr1
-
- %0:av_64_align2 = COPY $vgpr0_vgpr1
- %1:vgpr_32 = COPY $vgpr2
- %2:sreg_64 = COPY $sgpr0_sgpr1
- $vcc = COPY %2
- S_CBRANCH_VCCNZ %bb.2, implicit $vcc
- S_BRANCH %bb.1
-
- bb.1:
- successors: %bb.2
- %3:vgpr_32 = V_ADD_U32_e32 %1, %1, implicit $exec
-
- bb.2:
- %4:vgpr_32 = PHI %0.sub1, %bb.0, %3, %bb.1
- $vgpr0 = COPY %4
- SI_RETURN implicit $vgpr0
-...
-
-# The common subclass of an av_32 operand and an agpr_32 PHI is an AGPR class.
-# The copy is kept.
----
-name: phi_agpr_av_operand
-tracksRegLiveness: true
-body: |
- ; CHECK-LABEL: name: phi_agpr_av_operand
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; CHECK-NEXT: liveins: $vgpr0, $agpr0, $sgpr0_sgpr1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:av_32 = COPY $vgpr0
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:agpr_32 = COPY $agpr0
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
- ; CHECK-NEXT: $vcc = COPY [[COPY2]]
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr_32 = COPY [[COPY]], implicit $exec
- ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
- ; CHECK-NEXT: S_BRANCH %bb.1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:agpr_32 = COPY [[COPY1]]
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:agpr_32 = PHI [[COPY3]], %bb.0, [[COPY4]], %bb.1
- ; CHECK-NEXT: $agpr0 = COPY [[PHI]]
- ; CHECK-NEXT: SI_RETURN implicit $agpr0
- bb.0:
- successors: %bb.1, %bb.2
- liveins: $vgpr0, $agpr0, $sgpr0_sgpr1
-
- %0:av_32 = COPY $vgpr0
- %1:agpr_32 = COPY $agpr0
- %2:sreg_64 = COPY $sgpr0_sgpr1
- $vcc = COPY %2
- S_CBRANCH_VCCNZ %bb.2, implicit $vcc
- S_BRANCH %bb.1
-
- bb.1:
- successors: %bb.2
- %3:agpr_32 = COPY %1
-
- bb.2:
- %4:agpr_32 = PHI %0, %bb.0, %3, %bb.1
- $agpr0 = COPY %4
- SI_RETURN implicit $agpr0
-...
-
-# The same with an agpr_32 operand for an av_32 PHI. The copy is kept.
----
-name: phi_av_agpr_operand
-tracksRegLiveness: true
-body: |
- ; CHECK-LABEL: name: phi_av_agpr_operand
- ; CHECK: bb.0:
- ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
- ; CHECK-NEXT: liveins: $vgpr0, $agpr0, $sgpr0_sgpr1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:agpr_32 = COPY $agpr0
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_32 = COPY $vgpr0
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
- ; CHECK-NEXT: $vcc = COPY [[COPY2]]
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:av_32 = COPY [[COPY]], implicit $exec
- ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
- ; CHECK-NEXT: S_BRANCH %bb.1
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.1:
- ; CHECK-NEXT: successors: %bb.2(0x80000000)
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:av_32 = COPY [[COPY1]]
- ; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2:
- ; CHECK-NEXT: [[PHI:%[0-9]+]]:av_32 = PHI [[COPY3]], %bb.0, [[COPY4]], %bb.1
- ; CHECK-NEXT: $vgpr0 = COPY [[PHI]]
- ; CHECK-NEXT: SI_RETURN implicit $vgpr0
- bb.0:
- successors: %bb.1, %bb.2
- liveins: $vgpr0, $agpr0, $sgpr0_sgpr1
-
- %0:agpr_32 = COPY $agpr0
- %1:av_32 = COPY $vgpr0
- %2:sreg_64 = COPY $sgpr0_sgpr1
- $vcc = COPY %2
- S_CBRANCH_VCCNZ %bb.2, implicit $vcc
- S_BRANCH %bb.1
-
- bb.1:
- successors: %bb.2
- %3:av_32 = COPY %1
-
- bb.2:
- %4:av_32 = PHI %0, %bb.0, %3, %bb.1
- $vgpr0 = COPY %4
- SI_RETURN implicit $vgpr0
-...
>From 10cd916166a92bbc0f547d851ac7177364671557 Mon Sep 17 00:00:00 2001
From: Pablo Reble <pablo.reble at amd.com>
Date: Fri, 25 Sep 2026 14:52:46 -0500
Subject: [PATCH 3/3] Only legalize PHIs in SIFixSGPRCopies that need it
---
llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp | 13 +-
.../test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll | 773 +++++++++---------
.../CodeGen/AMDGPU/a-v-global-atomicrmw.ll | 504 ++++++------
llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll | 44 +-
.../AMDGPU/av-split-dead-valno-crash.ll | 43 +-
.../CodeGen/AMDGPU/copy-to-reg-frameindex.ll | 3 +-
.../test/CodeGen/AMDGPU/flat-saddr-atomics.ll | 52 +-
.../CodeGen/AMDGPU/insert-waitcnts-merge.ll | 2 +-
.../CodeGen/AMDGPU/local-atomicrmw-fadd.ll | 32 +-
.../CodeGen/AMDGPU/local-atomicrmw-fmax.ll | 64 +-
.../CodeGen/AMDGPU/local-atomicrmw-fmin.ll | 64 +-
.../CodeGen/AMDGPU/local-atomicrmw-fsub.ll | 64 +-
llvm/test/CodeGen/AMDGPU/phi-av-pressure.ll | 2 +-
.../rewrite-mfma-form-spill-cost-reset.ll | 218 ++---
.../si-fix-sgpr-copies-phi-av-classes.mir | 377 +++++++++
15 files changed, 1318 insertions(+), 937 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/si-fix-sgpr-copies-phi-av-classes.mir
diff --git a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
index 964ddde0fbb017..48623b14bb4785 100644
--- a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
@@ -862,8 +862,17 @@ void SIFixSGPRCopies::processPHINode(MachineInstr &MI) {
}
}
- if (TRI->hasVectorRegisters(MRI->getRegClass(PHIRes)) ||
- RC0 == &AMDGPU::VReg_1RegClass) {
+ // Only legalize PHIs that need it: i1 PHIs, PHIs just moved to AGPRs, and
+ // vector PHIs with SGPR or subregister inputs, which are left behind by
+ // SGPR-to-VGPR lowering (e.g. for temporal divergence).
+ const TargetRegisterClass *RC = MRI->getRegClass(PHIRes);
+ bool HasSALUInput =
+ any_of(drop_begin(MI.operands()), [&](const MachineOperand &MO) {
+ return MO.isReg() &&
+ (MO.getSubReg() || TRI->isSGPRReg(*MRI, MO.getReg()));
+ });
+ if (RC0 == &AMDGPU::VReg_1RegClass ||
+ (TRI->hasVectorRegisters(RC) && (RC != RC0 || HasSALUInput))) {
LLVM_DEBUG(dbgs() << "Legalizing PHI: " << MI);
TII->legalizeOperands(MI, MDT);
}
diff --git a/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll b/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll
index cb59bb925aab0b..0d32dd3fe5d268 100644
--- a/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll
+++ b/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll
@@ -11812,27 +11812,27 @@ define void @flat_atomic_fmaximum_v2f16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
+; GFX90A-NEXT: s_mov_b32 s6, 0x5040100
; GFX90A-NEXT: .LBB147_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v2, v3, v4
+; GFX90A-NEXT: v_cmp_o_f16_e32 vcc, v3, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v2, vcc
; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v3, v4 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v3, v4
-; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v2, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_sdwa v2, v5, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s8
+; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s6
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB147_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -11950,27 +11950,27 @@ define void @flat_atomic_fminimum_v2f16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
+; GFX90A-NEXT: s_mov_b32 s6, 0x5040100
; GFX90A-NEXT: .LBB149_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_min_f16 v2, v3, v4
+; GFX90A-NEXT: v_cmp_o_f16_e32 vcc, v3, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v2, vcc
; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v3, v4 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v3, v4
-; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v2, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_sdwa v2, v5, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s8
+; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s6
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB149_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -12092,38 +12092,38 @@ define void @flat_atomic_fadd_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB151_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s7
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB151_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -12225,38 +12225,38 @@ define void @flat_atomic_fsub_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB153_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s7
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB153_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -12397,38 +12397,38 @@ define void @flat_atomic_fmax_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB155_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s7
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB155_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -12569,38 +12569,38 @@ define void @flat_atomic_fmin_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB157_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s7
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB157_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -12741,43 +12741,43 @@ define void @flat_atomic_fmaximum_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB159_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_max_f32_e32 v7, v2, v4
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v2, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_max_f32_e32 v8, v2, v4
-; GFX90A-NEXT: v_max_f32_e32 v9, v7, v6
+; GFX90A-NEXT: v_max_f32_e32 v8, v7, v6
; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v4
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
-; GFX90A-NEXT: v_bfe_u32 v8, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX90A-NEXT: v_add3_u32 v8, v8, v2, s8
-; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v8, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v7, 16, 1
+; GFX90A-NEXT: v_add3_u32 v8, v8, v7, s6
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v8, v9, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s7
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB159_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -12923,43 +12923,43 @@ define void @flat_atomic_fminimum_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB161_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_min_f32_e32 v7, v2, v4
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v2, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_min_f32_e32 v8, v2, v4
-; GFX90A-NEXT: v_min_f32_e32 v9, v7, v6
+; GFX90A-NEXT: v_min_f32_e32 v8, v7, v6
; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v4
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
-; GFX90A-NEXT: v_bfe_u32 v8, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX90A-NEXT: v_add3_u32 v8, v8, v2, s8
-; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v8, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v7, 16, 1
+; GFX90A-NEXT: v_add3_u32 v8, v8, v7, s6
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v8, v9, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s7
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB161_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -13490,28 +13490,27 @@ define void @flat_atomic_nand_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB172_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_and_b32_e32 v0, v1, v4
-; GFX90A-NEXT: v_not_b32_e32 v0, v0
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_and_b32_e32 v2, v3, v4
+; GFX90A-NEXT: v_not_b32_e32 v2, v2
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB172_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; use v0
+; GFX90A-NEXT: ; use v2
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -13519,9 +13518,8 @@ define void @flat_atomic_nand_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
@@ -13529,18 +13527,18 @@ define void @flat_atomic_nand_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: .LBB172_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_bitop3_b32 v0, v1, v4, v1 bitop3:0x3f
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_bitop3_b32 v2, v3, v4, v3 bitop3:0x3f
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB172_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v0
+; GFX950-NEXT: ; use v2
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 10
@@ -14259,29 +14257,28 @@ define void @flat_atomic_usub_cond_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB190_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_sub_u32_e32 v0, v1, v4
-; GFX90A-NEXT: v_cmp_ge_u32_e32 vcc, v1, v4
-; GFX90A-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_sub_u32_e32 v2, v3, v4
+; GFX90A-NEXT: v_cmp_ge_u32_e32 vcc, v3, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB190_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; use v0
+; GFX90A-NEXT: ; use v2
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -14289,30 +14286,29 @@ define void @flat_atomic_usub_cond_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: .LBB190_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_sub_u32_e32 v0, v1, v4
-; GFX950-NEXT: v_cmp_ge_u32_e32 vcc, v1, v4
+; GFX950-NEXT: v_sub_u32_e32 v2, v3, v4
+; GFX950-NEXT: v_cmp_ge_u32_e32 vcc, v3, v4
; GFX950-NEXT: s_nop 1
-; GFX950-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB190_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v0
+; GFX950-NEXT: ; use v2
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 10
@@ -14395,27 +14391,26 @@ define void @flat_atomic_usub_sat_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB192_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_sub_u32_e64 v0, v1, v4 clamp
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_sub_u32_e64 v2, v3, v4 clamp
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB192_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; use v0
+; GFX90A-NEXT: ; use v2
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -14423,9 +14418,8 @@ define void @flat_atomic_usub_sat_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
@@ -14433,18 +14427,18 @@ define void @flat_atomic_usub_sat_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: .LBB192_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_sub_u32_e64 v0, v1, v4 clamp
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_sub_u32_e64 v2, v3, v4 clamp
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB192_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v0
+; GFX950-NEXT: ; use v2
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x i32], ptr %ptr, i64 0, i64 10
@@ -17140,18 +17134,18 @@ define void @flat_atomic_udec_wrap_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v[2:3]
+; GFX950-NEXT: ; def v[0:1]
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_cbranch_scc0 .LBB218_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] sc0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-NEXT: flat_atomic_dec_x2 v[2:3], v[2:3], v[0:1] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: s_branch .LBB218_3
; GFX950-NEXT: .LBB218_2:
; GFX950-NEXT: s_mov_b64 s[2:3], -1
-; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-NEXT: .LBB218_3: ; %Flow
; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: s_cselect_b32 s2, 1, 0
@@ -17160,21 +17154,21 @@ define void @flat_atomic_udec_wrap_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s2, s0, -1
-; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s2
+; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s2
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_subrev_co_u32_e32 v4, vcc, 1, v0
-; GFX950-NEXT: v_cmp_gt_u64_e64 s[0:1], v[0:1], v[2:3]
+; GFX950-NEXT: v_subrev_co_u32_e32 v4, vcc, 1, v2
+; GFX950-NEXT: v_cmp_gt_u64_e64 s[0:1], v[2:3], v[0:1]
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_subbrev_co_u32_e32 v5, vcc, 0, v1, vcc
+; GFX950-NEXT: v_subbrev_co_u32_e32 v5, vcc, 0, v3, vcc
; GFX950-NEXT: s_or_b64 vcc, vcc, s[0:1]
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX950-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s2
+; GFX950-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc
+; GFX950-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s2
; GFX950-NEXT: .LBB218_5: ; %atomicrmw.phi
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v[0:1]
+; GFX950-NEXT: ; use v[2:3]
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x i64], ptr %ptr, i64 0, i64 10
@@ -17965,27 +17959,26 @@ define void @flat_atomic_fsub_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB226_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_sub_f32_e32 v0, v1, v4
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_sub_f32_e32 v2, v3, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB226_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; use v0
+; GFX90A-NEXT: ; use v2
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -17993,27 +17986,26 @@ define void @flat_atomic_fsub_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: .LBB226_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_sub_f32_e32 v0, v1, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_sub_f32_e32 v2, v3, v4
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB226_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v0
+; GFX950-NEXT: ; use v2
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 10
@@ -18406,9 +18398,8 @@ define void @flat_atomic_fmaximum_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
@@ -18416,18 +18407,18 @@ define void @flat_atomic_fmaximum_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: .LBB232_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_maximum3_f32 v0, v1, v4, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_maximum3_f32 v2, v3, v4, v4
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB232_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v0
+; GFX950-NEXT: ; use v2
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 10
@@ -18544,9 +18535,8 @@ define void @flat_atomic_fminimum_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
@@ -18554,18 +18544,18 @@ define void @flat_atomic_fminimum_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: .LBB234_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_minimum3_f32 v0, v1, v4, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_minimum3_f32 v2, v3, v4, v4
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB234_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v0
+; GFX950-NEXT: ; use v2
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 10
@@ -19264,18 +19254,18 @@ define void @flat_atomic_fmax_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v[2:3]
+; GFX950-NEXT: ; def v[0:1]
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_cbranch_scc0 .LBB240_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_atomic_max_f64 v[0:1], v[0:1], v[2:3] sc0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-NEXT: flat_atomic_max_f64 v[2:3], v[2:3], v[0:1] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: s_branch .LBB240_3
; GFX950-NEXT: .LBB240_2:
; GFX950-NEXT: s_mov_b64 s[2:3], -1
-; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-NEXT: .LBB240_3: ; %Flow
; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: s_cselect_b32 s2, 1, 0
@@ -19284,16 +19274,16 @@ define void @flat_atomic_fmax_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
-; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s0
-; GFX950-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s0
+; GFX950-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
-; GFX950-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]
-; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s0
+; GFX950-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX950-NEXT: v_max_f64 v[0:1], v[4:5], v[0:1]
+; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s0
; GFX950-NEXT: .LBB240_5: ; %atomicrmw.end
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v[0:1]
+; GFX950-NEXT: ; use v[2:3]
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 10
@@ -19470,18 +19460,18 @@ define void @flat_atomic_fmin_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_cselect_b32 s2, 1, 0
; GFX950-NEXT: s_cmp_lg_u32 s2, 1
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v[2:3]
+; GFX950-NEXT: ; def v[0:1]
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_cbranch_scc0 .LBB242_2
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_atomic_min_f64 v[0:1], v[0:1], v[2:3] sc0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
+; GFX950-NEXT: flat_atomic_min_f64 v[2:3], v[2:3], v[0:1] sc0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: s_branch .LBB242_3
; GFX950-NEXT: .LBB242_2:
; GFX950-NEXT: s_mov_b64 s[2:3], -1
-; GFX950-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX950-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX950-NEXT: .LBB242_3: ; %Flow
; GFX950-NEXT: s_and_b64 s[2:3], s[2:3], exec
; GFX950-NEXT: s_cselect_b32 s2, 1, 0
@@ -19490,16 +19480,16 @@ define void @flat_atomic_fmin_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; %bb.4: ; %atomicrmw.private
; GFX950-NEXT: s_cmp_lg_u64 s[0:1], 0
; GFX950-NEXT: s_cselect_b32 s0, s0, -1
-; GFX950-NEXT: scratch_load_dwordx2 v[0:1], off, s0
-; GFX950-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
+; GFX950-NEXT: scratch_load_dwordx2 v[2:3], off, s0
+; GFX950-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
-; GFX950-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]
-; GFX950-NEXT: scratch_store_dwordx2 off, v[2:3], s0
+; GFX950-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX950-NEXT: v_min_f64 v[0:1], v[4:5], v[0:1]
+; GFX950-NEXT: scratch_store_dwordx2 off, v[0:1], s0
; GFX950-NEXT: .LBB242_5: ; %atomicrmw.end
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v[0:1]
+; GFX950-NEXT: ; use v[2:3]
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x double], ptr %ptr, i64 0, i64 10
@@ -20095,27 +20085,26 @@ define void @flat_atomic_fadd_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB248_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_add_f16 v0, v1, v4
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_pk_add_f16 v2, v3, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB248_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; use v0
+; GFX90A-NEXT: ; use v2
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -20213,27 +20202,26 @@ define void @flat_atomic_fsub_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB250_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_add_f16 v0, v1, v4 neg_lo:[0,1] neg_hi:[0,1]
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_pk_add_f16 v2, v3, v4 neg_lo:[0,1] neg_hi:[0,1]
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v1, v0
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB250_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; use v0
+; GFX90A-NEXT: ; use v2
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -20241,9 +20229,8 @@ define void @flat_atomic_fsub_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
@@ -20251,18 +20238,18 @@ define void @flat_atomic_fsub_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: .LBB250_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_add_f16 v0, v1, v4 neg_lo:[0,1] neg_hi:[0,1]
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_pk_add_f16 v2, v3, v4 neg_lo:[0,1] neg_hi:[0,1]
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB250_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v0
+; GFX950-NEXT: ; use v2
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 10
@@ -20566,28 +20553,28 @@ define void @flat_atomic_fmaximum_v2f16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
+; GFX90A-NEXT: s_mov_b32 s6, 0x5040100
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB255_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v0, v1, v4
+; GFX90A-NEXT: v_cmp_o_f16_e32 vcc, v1, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v0, vcc
; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v1, v4 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v1, v4
-; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v0, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_sdwa v0, v5, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_perm_b32 v0, v0, v6, s8
+; GFX90A-NEXT: v_perm_b32 v0, v0, v6, s6
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB255_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -20671,9 +20658,8 @@ define void @flat_atomic_fmaximum_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
@@ -20681,18 +20667,18 @@ define void @flat_atomic_fmaximum_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: .LBB256_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_maximum3_f16 v0, v1, v4, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_pk_maximum3_f16 v2, v3, v4, v4
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB256_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v0
+; GFX950-NEXT: ; use v2
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 10
@@ -20712,28 +20698,28 @@ define void @flat_atomic_fminimum_v2f16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
+; GFX90A-NEXT: s_mov_b32 s6, 0x5040100
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB257_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_min_f16 v0, v1, v4
+; GFX90A-NEXT: v_cmp_o_f16_e32 vcc, v1, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v0, vcc
; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v1, v4 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v1, v4
-; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v0, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_sdwa v0, v5, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_perm_b32 v0, v0, v6, s8
+; GFX90A-NEXT: v_perm_b32 v0, v0, v6, s6
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB257_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -20817,9 +20803,8 @@ define void @flat_atomic_fminimum_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
@@ -20827,18 +20812,18 @@ define void @flat_atomic_fminimum_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: .LBB258_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_minimum3_f16 v0, v1, v4, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_pk_minimum3_f16 v2, v3, v4, v4
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB258_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; use v0
+; GFX950-NEXT: ; use v2
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 10
@@ -20862,39 +20847,39 @@ define void @flat_atomic_fadd_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB259_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
; GFX90A-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s7
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB259_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -21003,39 +20988,39 @@ define void @flat_atomic_fsub_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB261_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
; GFX90A-NEXT: v_sub_f32_e32 v0, v0, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s7
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB261_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -21183,39 +21168,39 @@ define void @flat_atomic_fmax_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB263_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
; GFX90A-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s7
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB263_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -21363,39 +21348,39 @@ define void @flat_atomic_fmin_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB265_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
; GFX90A-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s7
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB265_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -21543,44 +21528,44 @@ define void @flat_atomic_fmaximum_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB267_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_max_f32_e32 v7, v0, v4
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v0, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v7, vcc
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc
; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_max_f32_e32 v8, v0, v4
-; GFX90A-NEXT: v_max_f32_e32 v9, v7, v6
+; GFX90A-NEXT: v_max_f32_e32 v8, v7, v6
; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v0, v4
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
-; GFX90A-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX90A-NEXT: v_add3_u32 v8, v8, v0, s8
-; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v8, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v7, 16, 1
+; GFX90A-NEXT: v_add3_u32 v8, v8, v7, s6
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v8, v9, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v7, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v7, v0, s7
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB267_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -21733,44 +21718,44 @@ define void @flat_atomic_fminimum_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB269_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_min_f32_e32 v7, v0, v4
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v0, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v7, vcc
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc
; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_min_f32_e32 v8, v0, v4
-; GFX90A-NEXT: v_min_f32_e32 v9, v7, v6
+; GFX90A-NEXT: v_min_f32_e32 v8, v7, v6
; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v0, v4
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
-; GFX90A-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX90A-NEXT: v_add3_u32 v8, v8, v0, s8
-; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v8, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v7, 16, 1
+; GFX90A-NEXT: v_add3_u32 v8, v8, v7, s6
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v8, v9, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v7, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v7, v0, s7
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB269_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
diff --git a/llvm/test/CodeGen/AMDGPU/a-v-global-atomicrmw.ll b/llvm/test/CodeGen/AMDGPU/a-v-global-atomicrmw.ll
index 4983695ac24ee9..304a8501b904d9 100644
--- a/llvm/test/CodeGen/AMDGPU/a-v-global-atomicrmw.ll
+++ b/llvm/test/CodeGen/AMDGPU/a-v-global-atomicrmw.ll
@@ -7884,27 +7884,27 @@ define void @global_atomic_fmaximum_v2f16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
+; GFX90A-NEXT: s_mov_b32 s6, 0x5040100
; GFX90A-NEXT: .LBB147_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v2, v3, v4
+; GFX90A-NEXT: v_cmp_o_f16_e32 vcc, v3, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v2, vcc
; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v3, v4 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v3, v4
-; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v2, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_sdwa v2, v5, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s8
+; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s6
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB147_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -8022,27 +8022,27 @@ define void @global_atomic_fminimum_v2f16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
+; GFX90A-NEXT: s_mov_b32 s6, 0x5040100
; GFX90A-NEXT: .LBB149_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_min_f16 v2, v3, v4
+; GFX90A-NEXT: v_cmp_o_f16_e32 vcc, v3, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v2, vcc
; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v3, v4 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v3, v4
-; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v2, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_sdwa v2, v5, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s8
+; GFX90A-NEXT: v_perm_b32 v2, v2, v6, s6
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB149_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -8164,38 +8164,38 @@ define void @global_atomic_fadd_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB151_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s7
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB151_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -8297,38 +8297,38 @@ define void @global_atomic_fsub_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB153_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s7
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB153_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -8469,38 +8469,38 @@ define void @global_atomic_fmax_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB155_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s7
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB155_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -8641,38 +8641,38 @@ define void @global_atomic_fmin_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB157_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s7
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB157_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -8813,43 +8813,43 @@ define void @global_atomic_fmaximum_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB159_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_max_f32_e32 v7, v2, v4
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v2, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_max_f32_e32 v8, v2, v4
-; GFX90A-NEXT: v_max_f32_e32 v9, v7, v6
+; GFX90A-NEXT: v_max_f32_e32 v8, v7, v6
; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v4
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
-; GFX90A-NEXT: v_bfe_u32 v8, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX90A-NEXT: v_add3_u32 v8, v8, v2, s8
-; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v8, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v7, 16, 1
+; GFX90A-NEXT: v_add3_u32 v8, v8, v7, s6
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v8, v9, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s7
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB159_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -8995,43 +8995,43 @@ define void @global_atomic_fminimum_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB161_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_min_f32_e32 v7, v2, v4
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v2, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_min_f32_e32 v8, v2, v4
-; GFX90A-NEXT: v_min_f32_e32 v9, v7, v6
+; GFX90A-NEXT: v_min_f32_e32 v8, v7, v6
; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v4
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
-; GFX90A-NEXT: v_bfe_u32 v8, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX90A-NEXT: v_add3_u32 v8, v8, v2, s8
-; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v8, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v7, 16, 1
+; GFX90A-NEXT: v_add3_u32 v8, v8, v7, s6
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v7
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v8, v9, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s7
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB161_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v2
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -13763,27 +13763,27 @@ define void @global_atomic_fmaximum_v2f16_saddr_ret_a_a(ptr addrspace(1) inreg %
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v3, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_mov_b32_e32 v4, 0x7e00
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
+; GFX90A-NEXT: s_mov_b32 s6, 0x5040100
; GFX90A-NEXT: .LBB255_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_max_f16 v0, v1, v3
+; GFX90A-NEXT: v_cmp_o_f16_e32 vcc, v1, v3
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v4, v0, vcc
; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v1, v3 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v1, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v5, v4, v0, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_sdwa v0, v4, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_perm_b32 v0, v0, v5, s8
+; GFX90A-NEXT: v_perm_b32 v0, v0, v5, s6
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB255_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -13905,27 +13905,27 @@ define void @global_atomic_fminimum_v2f16_saddr_ret_a_a(ptr addrspace(1) inreg %
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v3, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_mov_b32_e32 v4, 0x7e00
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
+; GFX90A-NEXT: s_mov_b32 s6, 0x5040100
; GFX90A-NEXT: .LBB257_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_pk_min_f16 v0, v1, v3
+; GFX90A-NEXT: v_cmp_o_f16_e32 vcc, v1, v3
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v4, v0, vcc
; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v1, v3 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v1, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v5, v4, v0, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_sdwa v0, v4, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_perm_b32 v0, v0, v5, s8
+; GFX90A-NEXT: v_perm_b32 v0, v0, v5, s6
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB257_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -14051,38 +14051,38 @@ define void @global_atomic_fadd_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg %ptr
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB259_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX90A-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s6
+; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX90A-NEXT: v_add_f32_e32 v5, v5, v4
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v5
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s7
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB259_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -14188,38 +14188,38 @@ define void @global_atomic_fsub_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg %ptr
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB261_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX90A-NEXT: v_sub_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s6
+; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v4
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v5
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s7
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB261_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -14364,38 +14364,38 @@ define void @global_atomic_fmax_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg %ptr
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB263_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX90A-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s6
+; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX90A-NEXT: v_max_f32_e32 v5, v5, v4
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v5
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s7
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB263_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -14540,38 +14540,38 @@ define void @global_atomic_fmin_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg %ptr
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB265_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX90A-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s6
+; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v6, vcc
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX90A-NEXT: v_min_f32_e32 v5, v5, v4
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v5, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v5
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s7
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB265_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -14716,43 +14716,43 @@ define void @global_atomic_fmaximum_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v4, 0x7fc00000
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB267_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_max_f32_e32 v6, v0, v3
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v0, v3
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX90A-NEXT: v_max_f32_e32 v7, v0, v3
-; GFX90A-NEXT: v_max_f32_e32 v8, v6, v5
+; GFX90A-NEXT: v_max_f32_e32 v7, v6, v5
; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v6, v5
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v0, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v4, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v4, v8, vcc
-; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v4, v7, vcc
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s7
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB267_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
@@ -14902,43 +14902,43 @@ define void @global_atomic_fminimum_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
-; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX90A-NEXT: v_mov_b32_e32 v4, 0x7fc00000
-; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: s_movk_i32 s6, 0x7fff
; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: s_mov_b32 s7, 0x7060302
; GFX90A-NEXT: .LBB269_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_min_f32_e32 v6, v0, v3
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v0, v3
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s6
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX90A-NEXT: v_min_f32_e32 v7, v0, v3
-; GFX90A-NEXT: v_min_f32_e32 v8, v6, v5
+; GFX90A-NEXT: v_min_f32_e32 v7, v6, v5
; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v6, v5
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v0, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v4, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v4, v8, vcc
-; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v4, v7, vcc
+; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s6
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s7
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: v_mov_b32_e32 v1, v0
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB269_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_accvgpr_write_b32 a0, v0
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; use a0
diff --git a/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll b/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll
index 59f7e5c2b43f7c..431416e6f917a8 100644
--- a/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomicrmw-expand.ll
@@ -775,7 +775,7 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX90A-NEXT: s_mov_b64 s[4:5], -1
; GFX90A-NEXT: s_xor_b64 s[6:7], s[6:7], s[4:5]
; GFX90A-NEXT: s_and_b64 vcc, exec, s[6:7]
-; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX90A-NEXT: s_cbranch_vccnz .LBB5_3
; GFX90A-NEXT: .LBB5_1: ; %Flow4
; GFX90A-NEXT: s_mov_b64 s[6:7], -1
@@ -783,7 +783,7 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX90A-NEXT: s_and_b64 vcc, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_vccnz .LBB5_10
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.shared
-; GFX90A-NEXT: ds_add_rtn_f64 v[2:3], v0, v[0:1]
+; GFX90A-NEXT: ds_add_rtn_f64 v[4:5], v0, v[0:1]
; GFX90A-NEXT: s_branch .LBB5_10
; GFX90A-NEXT: .LBB5_3: ; %atomicrmw.check.private
; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base
@@ -795,16 +795,16 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX90A-NEXT: s_mov_b64 s[4:5], -1
; GFX90A-NEXT: s_xor_b64 s[6:7], s[6:7], s[4:5]
; GFX90A-NEXT: s_and_b64 vcc, exec, s[6:7]
-; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX90A-NEXT: s_cbranch_vccnz .LBB5_5
; GFX90A-NEXT: s_branch .LBB5_6
; GFX90A-NEXT: .LBB5_4: ; %atomicrmw.private
-; GFX90A-NEXT: buffer_load_dword v2, v0, s[0:3], 0 offen
+; GFX90A-NEXT: buffer_load_dword v4, v0, s[0:3], 0 offen
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
-; GFX90A-NEXT: v_add_f64 v[4:5], v[2:3], v[0:1]
-; GFX90A-NEXT: buffer_store_dword v5, v0, s[0:3], 0 offen
-; GFX90A-NEXT: buffer_store_dword v4, v0, s[0:3], 0 offen
+; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1]
+; GFX90A-NEXT: buffer_store_dword v3, v0, s[0:3], 0 offen
+; GFX90A-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GFX90A-NEXT: s_branch .LBB5_9
; GFX90A-NEXT: .LBB5_5: ; %atomicrmw.global
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
@@ -829,8 +829,8 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX90A-NEXT: global_atomic_cmpswap_x2 v[2:3], v6, v[2:5], s[6:7] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u64_e64 s[6:7], v[2:3], v[4:5]
-; GFX90A-NEXT: s_or_b64 s[4:5], s[6:7], s[4:5]
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
+; GFX90A-NEXT: s_or_b64 s[4:5], s[6:7], s[4:5]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB5_7
; GFX90A-NEXT: ; %bb.8: ; %atomicrmw.end1
@@ -844,9 +844,9 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX90A-NEXT: ; %bb.11: ; %atomicrmw.end
; GFX90A-NEXT: s_mov_b32 s4, 32
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b64 v[4:5], s4, v[2:3]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v2
-; GFX90A-NEXT: v_mov_b32_e32 v1, v4
+; GFX90A-NEXT: v_lshrrev_b64 v[2:3], s4, v[4:5]
+; GFX90A-NEXT: v_mov_b32_e32 v0, v4
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
@@ -862,7 +862,7 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX942-NEXT: s_mov_b64 s[0:1], -1
; GFX942-NEXT: s_xor_b64 s[2:3], s[2:3], s[0:1]
; GFX942-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX942-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX942-NEXT: s_cbranch_vccnz .LBB5_3
; GFX942-NEXT: .LBB5_1: ; %Flow4
; GFX942-NEXT: s_mov_b64 s[2:3], -1
@@ -870,7 +870,7 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX942-NEXT: s_and_b64 vcc, exec, s[0:1]
; GFX942-NEXT: s_cbranch_vccnz .LBB5_10
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.shared
-; GFX942-NEXT: ds_add_rtn_f64 v[2:3], v0, v[0:1]
+; GFX942-NEXT: ds_add_rtn_f64 v[4:5], v0, v[0:1]
; GFX942-NEXT: s_branch .LBB5_10
; GFX942-NEXT: .LBB5_3: ; %atomicrmw.check.private
; GFX942-NEXT: s_mov_b64 s[0:1], src_private_base
@@ -882,14 +882,14 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX942-NEXT: s_mov_b64 s[0:1], -1
; GFX942-NEXT: s_xor_b64 s[2:3], s[2:3], s[0:1]
; GFX942-NEXT: s_and_b64 vcc, exec, s[2:3]
-; GFX942-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX942-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX942-NEXT: s_cbranch_vccnz .LBB5_5
; GFX942-NEXT: s_branch .LBB5_6
; GFX942-NEXT: .LBB5_4: ; %atomicrmw.private
-; GFX942-NEXT: scratch_load_dwordx2 v[2:3], off, s0
+; GFX942-NEXT: scratch_load_dwordx2 v[4:5], off, s0
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_add_f64 v[4:5], v[2:3], v[0:1]
-; GFX942-NEXT: scratch_store_dwordx2 off, v[4:5], s0
+; GFX942-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1]
+; GFX942-NEXT: scratch_store_dwordx2 off, v[2:3], s0
; GFX942-NEXT: s_branch .LBB5_9
; GFX942-NEXT: .LBB5_5: ; %atomicrmw.global
; GFX942-NEXT: v_mov_b32_e32 v2, 0
@@ -914,8 +914,8 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX942-NEXT: global_atomic_cmpswap_x2 v[2:3], v6, v[2:5], s[2:3] sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_cmp_eq_u64_e64 s[2:3], v[2:3], v[4:5]
-; GFX942-NEXT: s_or_b64 s[0:1], s[2:3], s[0:1]
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
+; GFX942-NEXT: s_or_b64 s[0:1], s[2:3], s[0:1]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB5_7
; GFX942-NEXT: ; %bb.8: ; %atomicrmw.end1
@@ -929,9 +929,9 @@ define double @optnone_atomicrmw_fadd_f64_expand(double %val) #1 {
; GFX942-NEXT: ; %bb.11: ; %atomicrmw.end
; GFX942-NEXT: s_mov_b32 s0, 32
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshrrev_b64 v[4:5], s0, v[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v2
-; GFX942-NEXT: v_mov_b32_e32 v1, v4
+; GFX942-NEXT: v_lshrrev_b64 v[2:3], s0, v[4:5]
+; GFX942-NEXT: v_mov_b32_e32 v0, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll b/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
index fbcba25ba207cf..1d4ada34650c3a 100644
--- a/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
@@ -7,14 +7,16 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: s_load_dword s0, s[4:5], 0x8
; CHECK-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x0
; CHECK-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x10
-; CHECK-NEXT: v_mov_b32_e32 v30, 0x9037ab78
-; CHECK-NEXT: v_mov_b32_e32 v31, 0x3e21eeb6
+; CHECK-NEXT: v_mov_b32_e32 v0, 0x9037ab78
+; CHECK-NEXT: v_mov_b32_e32 v1, 0x3e21eeb6
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_bitcmp1_b32 s0, 0
; CHECK-NEXT: s_cselect_b64 s[14:15], -1, 0
; CHECK-NEXT: s_xor_b64 s[16:17], s[14:15], -1
; CHECK-NEXT: s_bitcmp1_b32 s0, 8
+; CHECK-NEXT: v_accvgpr_write_b32 a4, v1 ; Reload Reuse
; CHECK-NEXT: s_cselect_b64 s[18:19], -1, 0
+; CHECK-NEXT: v_accvgpr_write_b32 a5, v0 ; Reload Reuse
; CHECK-NEXT: s_xor_b64 s[20:21], s[18:19], -1
; CHECK-NEXT: v_mov_b32_e32 v2, 0xa17f65f6
; CHECK-NEXT: v_mov_b32_e32 v3, 0xbe927e4f
@@ -32,14 +34,14 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: v_mov_b32_e32 v14, 0x8427b883
; CHECK-NEXT: v_mov_b32_e32 v15, 0x3fae1bb4
; CHECK-NEXT: s_and_b64 s[0:1], exec, s[14:15]
-; CHECK-NEXT: v_mov_b32_e32 v16, 0x57b87036
-; CHECK-NEXT: v_mov_b32_e32 v17, 0x3fb3b136
+; CHECK-NEXT: v_mov_b32_e32 v20, 0x57b87036
+; CHECK-NEXT: v_mov_b32_e32 v21, 0x3fb3b136
; CHECK-NEXT: s_and_b64 s[2:3], exec, s[18:19]
; CHECK-NEXT: v_mov_b32_e32 v18, 0x55555523
; CHECK-NEXT: v_mov_b32_e32 v19, 0xbfd55555
; CHECK-NEXT: s_and_b64 s[4:5], exec, s[16:17]
; CHECK-NEXT: v_mov_b32_e32 v0, 0
-; CHECK-NEXT: v_mov_b64_e32 v[20:21], 0
+; CHECK-NEXT: v_mov_b64_e32 v[30:31], 0
; CHECK-NEXT: ; implicit-def: $agpr0_agpr1
; CHECK-NEXT: ; implicit-def: $vgpr22_vgpr23
; CHECK-NEXT: s_branch .LBB0_2
@@ -64,8 +66,11 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: v_mov_b64_e32 v[24:25], s[10:11]
; CHECK-NEXT: flat_load_dwordx2 v[24:25], v[24:25]
-; CHECK-NEXT: v_mov_b64_e32 v[26:27], v[30:31]
+; CHECK-NEXT: v_accvgpr_read_b32 v27, a4 ; Reload Reuse
+; CHECK-NEXT: v_accvgpr_read_b32 v26, a5 ; Reload Reuse
; CHECK-NEXT: v_mov_b64_e32 v[28:29], v[2:3]
+; CHECK-NEXT: v_mov_b64_e32 v[16:17], v[20:21]
+; CHECK-NEXT: v_mov_b64_e32 v[30:31], 0
; CHECK-NEXT: v_accvgpr_write_b32 a2, 0
; CHECK-NEXT: v_accvgpr_write_b32 a3, 0
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -83,10 +88,9 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: v_fmac_f64_e32 v[26:27], 0, v[28:29]
; CHECK-NEXT: v_mov_b64_e32 v[28:29], v[14:15]
; CHECK-NEXT: v_fmac_f64_e32 v[28:29], 0, v[26:27]
-; CHECK-NEXT: v_mov_b64_e32 v[26:27], v[16:17]
-; CHECK-NEXT: v_fmac_f64_e32 v[26:27], 0, v[28:29]
-; CHECK-NEXT: v_mov_b64_e32 v[28:29], v[18:19]
-; CHECK-NEXT: v_fmac_f64_e32 v[28:29], 0, v[26:27]
+; CHECK-NEXT: v_fmac_f64_e32 v[16:17], 0, v[28:29]
+; CHECK-NEXT: v_mov_b64_e32 v[26:27], v[18:19]
+; CHECK-NEXT: v_fmac_f64_e32 v[26:27], 0, v[16:17]
; CHECK-NEXT: s_branch .LBB0_6
; CHECK-NEXT: .LBB0_5: ; %Flow
; CHECK-NEXT: ; in Loop: Header=BB0_6 Depth=2
@@ -95,8 +99,8 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: .LBB0_6: ; %.preheader1855.i.i.i3329
; CHECK-NEXT: ; Parent Loop BB0_2 Depth=1
; CHECK-NEXT: ; => This Inner Loop Header: Depth=2
-; CHECK-NEXT: v_accvgpr_read_b32 v27, a3
-; CHECK-NEXT: v_accvgpr_read_b32 v26, a2
+; CHECK-NEXT: v_accvgpr_read_b32 v29, a3
+; CHECK-NEXT: v_accvgpr_read_b32 v28, a2
; CHECK-NEXT: s_mov_b64 s[24:25], -1
; CHECK-NEXT: s_mov_b64 s[6:7], -1
; CHECK-NEXT: s_mov_b64 vcc, s[2:3]
@@ -112,8 +116,8 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: ; %bb.8: ; %.preheader1856.preheader.i.i.i3325
; CHECK-NEXT: ; in Loop: Header=BB0_6 Depth=2
; CHECK-NEXT: s_mov_b64 s[24:25], 0
-; CHECK-NEXT: v_accvgpr_write_b32 a2, v28
-; CHECK-NEXT: v_accvgpr_write_b32 a3, v29
+; CHECK-NEXT: v_accvgpr_write_b32 a2, v26
+; CHECK-NEXT: v_accvgpr_write_b32 a3, v27
; CHECK-NEXT: s_mov_b64 s[6:7], 0
; CHECK-NEXT: s_branch .LBB0_5
; CHECK-NEXT: .LBB0_9: ; in Loop: Header=BB0_2 Depth=1
@@ -133,16 +137,17 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: s_cbranch_vccz .LBB0_13
; CHECK-NEXT: ; %bb.12: ; %._crit_edge2105.i.i.i2330.loopexit
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: v_cmp_nlg_f64_e64 s[6:7], 0, v[26:27]
+; CHECK-NEXT: v_cmp_nlg_f64_e64 s[6:7], 0, v[28:29]
; CHECK-NEXT: v_cndmask_b32_e64 v23, v23, 0, s[14:15]
; CHECK-NEXT: v_cndmask_b32_e64 v22, v22, 0, s[14:15]
-; CHECK-NEXT: v_cndmask_b32_e64 v26, 0, 1, s[6:7]
-; CHECK-NEXT: v_mov_b32_e32 v27, v26
+; CHECK-NEXT: v_cndmask_b32_e64 v16, 0, 1, s[6:7]
+; CHECK-NEXT: v_mov_b32_e32 v17, v16
; CHECK-NEXT: s_and_b64 s[6:7], exec, s[14:15]
-; CHECK-NEXT: global_store_dwordx2 v0, v[26:27], s[8:9]
+; CHECK-NEXT: global_store_dwordx2 v0, v[16:17], s[8:9]
; CHECK-NEXT: s_cselect_b32 s23, s23, 0
; CHECK-NEXT: s_cselect_b32 s22, s22, 0
; CHECK-NEXT: s_mov_b64 s[6:7], -1
+; CHECK-NEXT: v_mov_b64_e32 v[30:31], 0
; CHECK-NEXT: s_branch .LBB0_14
; CHECK-NEXT: .LBB0_13: ; in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: s_mov_b64 s[6:7], 0
@@ -159,7 +164,7 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: ; %bb.16: ; %._crit_edge2105.i.i.i2330
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: s_mov_b64 s[24:25], 0
-; CHECK-NEXT: global_store_dwordx2 v0, v[20:21], s[8:9]
+; CHECK-NEXT: global_store_dwordx2 v0, v[30:31], s[8:9]
; CHECK-NEXT: s_branch .LBB0_1
; CHECK-NEXT: .LBB0_17: ; %DummyReturnBlock
; CHECK-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/copy-to-reg-frameindex.ll b/llvm/test/CodeGen/AMDGPU/copy-to-reg-frameindex.ll
index f7e9b8724b6821..04c40e7ccdaf6b 100644
--- a/llvm/test/CodeGen/AMDGPU/copy-to-reg-frameindex.ll
+++ b/llvm/test/CodeGen/AMDGPU/copy-to-reg-frameindex.ll
@@ -43,10 +43,11 @@ define void @phi_with_alloca_and_divergent_copy_to_reg(ptr addrspace(5) %diverge
; CHECK-LABEL: phi_with_alloca_and_divergent_copy_to_reg:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_lshr_b32 s6, s32, 6
; CHECK-NEXT: v_mov_b32_e32 v7, v2
; CHECK-NEXT: v_mov_b32_e32 v6, v1
; CHECK-NEXT: s_mov_b64 s[4:5], 0
-; CHECK-NEXT: v_lshrrev_b32_e64 v2, 6, s32
+; CHECK-NEXT: v_mov_b32_e32 v2, s6
; CHECK-NEXT: .LBB1_1: ; %loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: v_mov_b32_e32 v1, v2
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
index f40b209b77da7a..bb2dc353ab9fa5 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
@@ -14644,19 +14644,20 @@ define void @flat_atomic_fmax_f32_saddr_nortn(ptr inreg %ptr, float %data) {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[2:3] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: v_max_f32_e32 v4, v0, v0
+; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: .LBB119_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX950-NEXT: v_max_f32_e32 v0, v0, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX950-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB119_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14720,19 +14721,20 @@ define void @flat_atomic_fmin_f32_saddr_nortn(ptr inreg %ptr, float %data) {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[2:3] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: v_max_f32_e32 v4, v0, v0
+; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: .LBB121_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX950-NEXT: v_min_f32_e32 v0, v0, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX950-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB121_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14894,21 +14896,22 @@ define void @flat_atomic_fmax_v2f16_saddr_nortn(ptr inreg %ptr, <2 x half> %data
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[2:3] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: v_pk_max_f16 v4, v0, v0
+; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB125_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX950-NEXT: v_pk_max_f16 v2, v3, v3
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_max_f16 v0, v0, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_pk_max_f16 v2, v2, v4
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB125_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -15006,21 +15009,22 @@ define void @flat_atomic_fmin_v2f16_saddr_nortn(ptr inreg %ptr, <2 x half> %data
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
+; GFX950-NEXT: flat_load_dword v3, v[2:3] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: v_pk_max_f16 v4, v0, v0
+; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB127_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX950-NEXT: v_pk_max_f16 v2, v3, v3
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_min_f16 v0, v0, v4
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_pk_min_f16 v2, v2, v4
+; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v1, v0
+; GFX950-NEXT: v_mov_b32_e32 v3, v2
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB127_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/insert-waitcnts-merge.ll b/llvm/test/CodeGen/AMDGPU/insert-waitcnts-merge.ll
index fadb15dd2fb799..03d1abcb053719 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-waitcnts-merge.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert-waitcnts-merge.ll
@@ -23,9 +23,9 @@ define amdgpu_kernel void @widget(ptr addrspace(1) %arg, i1 %arg1) #0 {
; CHECK-NEXT: S_WAITCNT .Lgkmcnt_0
; CHECK-NEXT: S_BITCMP1_B32 killed renamable $sgpr2, 0, implicit-def $scc
; CHECK-NEXT: renamable $sgpr4_sgpr5 = S_MOV_B64 0
- ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
; CHECK-NEXT: renamable $sgpr2_sgpr3 = S_CSELECT_B64 -1, 0, implicit killed $scc
; CHECK-NEXT: $vgpr2_vgpr3 = V_PK_MOV_B32 8, 0, 8, 0, 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; CHECK-NEXT: renamable $sgpr6_sgpr7 = IMPLICIT_DEF
; CHECK-NEXT: S_BRANCH %bb.1
; CHECK-NEXT: {{ $}}
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
index e5e07ff1a94eb1..571c8aa6631c75 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
@@ -6430,19 +6430,19 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-LABEL: local_atomic_fadd_ret_v2bf16:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v3, v0
+; GFX90A-NEXT: ds_read_b32 v4, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX90A-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX90A-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_add_f32_e32 v5, v5, v3
; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6454,11 +6454,11 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v1
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6791,19 +6791,19 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-LABEL: local_atomic_fadd_ret_v2bf16__offset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v3, v0 offset:65532
+; GFX90A-NEXT: ds_read_b32 v4, v0 offset:65532
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX90A-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX90A-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_add_f32_e32 v5, v5, v3
; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6815,11 +6815,11 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1 offset:65532
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v1
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB25_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
index b74f6b1c346012..671a8c49025bba 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
@@ -6157,19 +6157,19 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX942-LABEL: local_atomic_fmax_ret_v2bf16:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: ds_read_b32 v3, v0
+; GFX942-NEXT: ds_read_b32 v4, v0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
-; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX942-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX942-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX942-NEXT: v_max_f32_e32 v5, v5, v3
; GFX942-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6182,11 +6182,11 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX942-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[0:1]
; GFX942-NEXT: v_perm_b32 v1, v5, v1, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v3, v1
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB24_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6329,19 +6329,19 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-LABEL: local_atomic_fmax_ret_v2bf16:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v3, v0
+; GFX90A-NEXT: ds_read_b32 v4, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX90A-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX90A-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_max_f32_e32 v5, v5, v3
; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6353,11 +6353,11 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v1
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6643,19 +6643,19 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX942-LABEL: local_atomic_fmax_ret_v2bf16__offset:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: ds_read_b32 v3, v0 offset:65532
+; GFX942-NEXT: ds_read_b32 v4, v0 offset:65532
; GFX942-NEXT: s_mov_b64 s[2:3], 0
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
-; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX942-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX942-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX942-NEXT: v_max_f32_e32 v5, v5, v3
; GFX942-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6668,11 +6668,11 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX942-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[0:1]
; GFX942-NEXT: v_perm_b32 v1, v5, v1, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1 offset:65532
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v3, v1
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB25_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6815,19 +6815,19 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-LABEL: local_atomic_fmax_ret_v2bf16__offset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v3, v0 offset:65532
+; GFX90A-NEXT: ds_read_b32 v4, v0 offset:65532
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX90A-NEXT: v_max_f32_e32 v1, v1, v2
-; GFX90A-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_max_f32_e32 v5, v5, v3
; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6839,11 +6839,11 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1 offset:65532
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v1
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB25_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
index 7c68c45a70a038..e9f671c44cffbe 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
@@ -6157,19 +6157,19 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX942-LABEL: local_atomic_fmin_ret_v2bf16:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: ds_read_b32 v3, v0
+; GFX942-NEXT: ds_read_b32 v4, v0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
-; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX942-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX942-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX942-NEXT: v_min_f32_e32 v5, v5, v3
; GFX942-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6182,11 +6182,11 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX942-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[0:1]
; GFX942-NEXT: v_perm_b32 v1, v5, v1, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v3, v1
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB24_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6329,19 +6329,19 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-LABEL: local_atomic_fmin_ret_v2bf16:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v3, v0
+; GFX90A-NEXT: ds_read_b32 v4, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX90A-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX90A-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_min_f32_e32 v5, v5, v3
; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6353,11 +6353,11 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v1
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6643,19 +6643,19 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX942-LABEL: local_atomic_fmin_ret_v2bf16__offset:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: ds_read_b32 v3, v0 offset:65532
+; GFX942-NEXT: ds_read_b32 v4, v0 offset:65532
; GFX942-NEXT: s_mov_b64 s[2:3], 0
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
-; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX942-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX942-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX942-NEXT: v_min_f32_e32 v5, v5, v3
; GFX942-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6668,11 +6668,11 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX942-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[0:1]
; GFX942-NEXT: v_perm_b32 v1, v5, v1, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1 offset:65532
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v3, v1
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB25_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6815,19 +6815,19 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-LABEL: local_atomic_fmin_ret_v2bf16__offset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v3, v0 offset:65532
+; GFX90A-NEXT: ds_read_b32 v4, v0 offset:65532
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX90A-NEXT: v_min_f32_e32 v1, v1, v2
-; GFX90A-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_min_f32_e32 v5, v5, v3
; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6839,11 +6839,11 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1 offset:65532
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v1
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB25_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
index 5c988fcda1a8e4..38723293fabb3d 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
@@ -6971,19 +6971,19 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX942-LABEL: local_atomic_fsub_ret_v2bf16:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: ds_read_b32 v3, v0
+; GFX942-NEXT: ds_read_b32 v4, v0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
-; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX942-NEXT: v_sub_f32_e32 v1, v1, v2
-; GFX942-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX942-NEXT: v_sub_f32_e32 v5, v5, v3
; GFX942-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -6996,11 +6996,11 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX942-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[0:1]
; GFX942-NEXT: v_perm_b32 v1, v5, v1, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v3, v1
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB24_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7143,19 +7143,19 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-LABEL: local_atomic_fsub_ret_v2bf16:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v3, v0
+; GFX90A-NEXT: ds_read_b32 v4, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX90A-NEXT: v_sub_f32_e32 v1, v1, v2
-; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v3
; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -7167,11 +7167,11 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v1
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7457,19 +7457,19 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX942-LABEL: local_atomic_fsub_ret_v2bf16__offset:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: ds_read_b32 v3, v0 offset:65532
+; GFX942-NEXT: ds_read_b32 v4, v0 offset:65532
; GFX942-NEXT: s_mov_b64 s[2:3], 0
; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
-; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX942-NEXT: v_sub_f32_e32 v1, v1, v2
-; GFX942-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX942-NEXT: v_sub_f32_e32 v5, v5, v3
; GFX942-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -7482,11 +7482,11 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX942-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[0:1]
; GFX942-NEXT: v_perm_b32 v1, v5, v1, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1 offset:65532
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v3, v1
+; GFX942-NEXT: v_mov_b32_e32 v4, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB25_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7629,19 +7629,19 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-LABEL: local_atomic_fsub_ret_v2bf16__offset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v3, v0 offset:65532
+; GFX90A-NEXT: ds_read_b32 v4, v0 offset:65532
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
-; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
; GFX90A-NEXT: v_sub_f32_e32 v1, v1, v2
-; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v3
; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
@@ -7653,11 +7653,11 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v4, v1 offset:65532
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v1
+; GFX90A-NEXT: v_mov_b32_e32 v4, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB25_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/phi-av-pressure.ll b/llvm/test/CodeGen/AMDGPU/phi-av-pressure.ll
index a4bf3705534bc4..29f26bb00135b3 100644
--- a/llvm/test/CodeGen/AMDGPU/phi-av-pressure.ll
+++ b/llvm/test/CodeGen/AMDGPU/phi-av-pressure.ll
@@ -36,9 +36,9 @@ define amdgpu_kernel void @main(i1 %arg, ptr %ptr, ptr addrspace(1) %ptr1, ptr a
; GFX950-NEXT: v_lshl_add_u64 v[10:11], v[0:1], 3, s[38:39]
; GFX950-NEXT: global_load_dwordx4 v[2:5], v[10:11], off
; GFX950-NEXT: global_load_dwordx4 v[6:9], v1, s[38:39]
-; GFX950-NEXT: v_mov_b32_e32 v0, 0
; GFX950-NEXT: v_mov_b64_e32 v[10:11], s[36:37]
; GFX950-NEXT: s_and_b64 vcc, exec, s[34:35]
+; GFX950-NEXT: v_mov_b32_e32 v0, v1
; GFX950-NEXT: v_mov_b32_e32 v12, v1
; GFX950-NEXT: v_mov_b32_e32 v14, v1
; GFX950-NEXT: v_mov_b32_e32 v15, v1
diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-spill-cost-reset.ll b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-spill-cost-reset.ll
index d115f9bd028f27..1b2b93f1e12d4c 100644
--- a/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-spill-cost-reset.ll
+++ b/llvm/test/CodeGen/AMDGPU/rewrite-mfma-form-spill-cost-reset.ll
@@ -68,86 +68,62 @@ define amdgpu_kernel void @test_spill_cost_reset(
; CHECK-NEXT: liveins: $sgpr4_sgpr5
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
- ; CHECK-NEXT: early-clobber %1891:sgpr_256 = S_LOAD_DWORDX8_IMM_ec [[COPY]](p4), 16, 0 :: (dereferenceable invariant load (s256) from %ir.a0.kernarg.offset, align 16, addrspace 4)
- ; CHECK-NEXT: early-clobber %1892:sgpr_256 = S_LOAD_DWORDX8_IMM_ec [[COPY]](p4), 48, 0 :: (dereferenceable invariant load (s256) from %ir.b0.kernarg.offset, align 16, addrspace 4)
- ; CHECK-NEXT: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]](p4), 80, 0 :: (dereferenceable invariant load (s32) from %ir.n.kernarg.offset, align 16, addrspace 4)
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_:%[0-9]+]].sub0:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: early-clobber %1843:sgpr_256 = S_LOAD_DWORDX8_IMM_ec [[COPY]](p4), 16, 0 :: (dereferenceable invariant load (s256) from %ir.a0.kernarg.offset, align 16, addrspace 4)
+ ; CHECK-NEXT: early-clobber %1844:sgpr_256 = S_LOAD_DWORDX8_IMM_ec [[COPY]](p4), 48, 0 :: (dereferenceable invariant load (s256) from %ir.b0.kernarg.offset, align 16, addrspace 4)
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec
; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_1:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_2:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_3:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_4:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_5:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_5:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_5:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_5:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_6:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_6:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_6:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_6:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_7:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_7:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_7:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_7:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_8:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_8:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_8:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_8:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_9:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_9:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_9:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_9:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_10:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_10:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_10:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_10:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_11:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_11:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_11:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_11:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: undef [[V_MOV_B32_e32_12:%[0-9]+]].sub0:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_12:%[0-9]+]].sub1:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_12:%[0-9]+]].sub2:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_12:%[0-9]+]].sub3:vreg_128_align2 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_128_align2 = COPY %1891.sub0_sub1_sub2_sub3
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_128_align2 = COPY %1892.sub0_sub1_sub2_sub3
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:av_128_align2 = COPY %1891.sub4_sub5_sub6_sub7
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:av_128_align2 = COPY %1892.sub4_sub5_sub6_sub7
- ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY5:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY5:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY6:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY6:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY7:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY7:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY8:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY8:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY9:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY9:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY10:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY10:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY11:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY11:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY12:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY12:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY13:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY13:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY14:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY14:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: undef [[COPY15:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
- ; CHECK-NEXT: [[COPY15:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]](p4), 80, 0 :: (dereferenceable invariant load (s32) from %ir.n.kernarg.offset, align 16, addrspace 4)
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_128_align2 = COPY %1843.sub0_sub1_sub2_sub3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_128_align2 = COPY %1844.sub0_sub1_sub2_sub3
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:av_128_align2 = COPY %1843.sub4_sub5_sub6_sub7
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:av_128_align2 = COPY %1844.sub4_sub5_sub6_sub7
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY5:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY6:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY7:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY8:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY9:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY10:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY11:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY12:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY13:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY14:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY15:%[0-9]+]].sub0:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]].sub1:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]].sub2:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]].sub3:vreg_128_align2 = COPY [[V_MOV_B32_e32_]].sub0
; CHECK-NEXT: undef [[COPY16:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
; CHECK-NEXT: [[COPY16:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
; CHECK-NEXT: undef [[COPY17:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
@@ -380,69 +356,105 @@ define amdgpu_kernel void @test_spill_cost_reset(
; CHECK-NEXT: [[COPY130:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
; CHECK-NEXT: undef [[COPY131:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
; CHECK-NEXT: [[COPY131:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY132:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY132:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY133:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY133:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY134:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY134:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY135:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY135:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY136:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY136:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY137:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY137:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY138:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY138:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY139:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY139:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY140:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY140:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY141:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY141:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY142:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY142:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: undef [[COPY143:%[0-9]+]].sub0:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
+ ; CHECK-NEXT: [[COPY143:%[0-9]+]].sub1:vreg_64_align2 = COPY [[V_MOV_B32_e32_]].sub0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1.loop:
; CHECK-NEXT: successors: %bb.2(0x04000000), %bb.1(0x7c000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[V_MOV_B32_e32_12:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_12]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY15]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY143:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY143]], 8, [[COPY143]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY142:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY142]], 8, [[COPY142]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY141:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY141]], 8, [[COPY141]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY140:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY140]], 8, [[COPY140]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY139:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY139]], 8, [[COPY139]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY14]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY138:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY138]], 8, [[COPY138]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY137:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY137]], 8, [[COPY137]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY136:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY136]], 8, [[COPY136]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY13]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY135:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY135]], 8, [[COPY135]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY134:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY134]], 8, [[COPY134]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY133:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY133]], 8, [[COPY133]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY12]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY132:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY132]], 8, [[COPY132]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY131:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY131]], 8, [[COPY131]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY130:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY130]], 8, [[COPY130]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY11]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY129:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY129]], 8, [[COPY129]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY128:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY128]], 8, [[COPY128]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY127:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY127]], 8, [[COPY127]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_11:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_11]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY10]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY126:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY126]], 8, [[COPY126]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY125:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY125]], 8, [[COPY125]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY124:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY124]], 8, [[COPY124]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_10:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_10]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY9]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY123:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY123]], 8, [[COPY123]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY122:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY122]], 8, [[COPY122]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY121:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY121]], 8, [[COPY121]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_9:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_9]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY8]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY120:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY120]], 8, [[COPY120]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY119:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY119]], 8, [[COPY119]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY118:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY118]], 8, [[COPY118]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_8:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_8]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY7]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY117:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY117]], 8, [[COPY117]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY116:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY116]], 8, [[COPY116]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY115:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY115]], 8, [[COPY115]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_7:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_7]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY6]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY114:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY114]], 8, [[COPY114]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY113:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY113]], 8, [[COPY113]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY112:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY112]], 8, [[COPY112]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_6:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_6]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[COPY5]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY111:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY111]], 8, [[COPY111]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY110:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY110]], 8, [[COPY110]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY109:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY109]], 8, [[COPY109]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_5:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_5]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY108:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY108]], 8, [[COPY108]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY107:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY107]], 8, [[COPY107]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY106:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY106]], 8, [[COPY106]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_4]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY105:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY105]], 8, [[COPY105]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY104:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY104]], 8, [[COPY104]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY103:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY103]], 8, [[COPY103]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_3]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY3]], [[COPY4]], [[COPY15]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_1:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY102:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY102]], 8, [[COPY102]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY101:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY101]], 8, [[COPY101]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY100:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY100]], 8, [[COPY100]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_2]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_2:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY3]], [[COPY4]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY99:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY99]], 8, [[COPY99]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY98:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY98]], 8, [[COPY98]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY97:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY97]], 8, [[COPY97]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MOV_B32_e32_1]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY96:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY96]], 8, [[COPY96]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY95:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY95]], 8, [[COPY95]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY94:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY94]], 8, [[COPY94]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY93:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY93]], 8, [[COPY93]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY92:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY92]], 8, [[COPY92]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY91:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY91]], 8, [[COPY91]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY3]], [[COPY4]], [[V_MOV_B32_e32_12]], 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_1:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY1]], [[COPY2]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY90:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY90]], 8, [[COPY90]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY89:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY89]], 8, [[COPY89]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY88:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY88]], 8, [[COPY88]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_2:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_F16_vgprcd_e64 [[COPY3]], [[COPY4]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY87:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY87]], 8, [[COPY87]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY86:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY86]], 8, [[COPY86]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY85:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY85]], 8, [[COPY85]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
@@ -514,28 +526,16 @@ define amdgpu_kernel void @test_spill_cost_reset(
; CHECK-NEXT: [[COPY19:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY19]], 8, [[COPY19]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY18:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY18]], 8, [[COPY18]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[COPY17:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY17]], 8, [[COPY17]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY16:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY16]], 8, [[COPY16]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY15:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY15]], 8, [[COPY15]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY14]], 8, [[COPY14]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY13]], 8, [[COPY13]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY12]], 8, [[COPY12]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY11]], 8, [[COPY11]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY10]], 8, [[COPY10]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY9]], 8, [[COPY9]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY8]], 8, [[COPY8]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY7]], 8, [[COPY7]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY6:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY6]], 8, [[COPY6]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY5]], 8, [[COPY5]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_MOV_B32_]], 1, implicit-def dead $scc
; CHECK-NEXT: S_CMP_LT_I32 [[S_MOV_B32_]], [[S_LOAD_DWORD_IMM]], implicit-def $scc
- ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[V_MOV_B32_e32_]], 8, [[V_MOV_B32_e32_]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY16:%[0-9]+]]:vreg_64_align2 = nofpexcept V_PK_ADD_F32 8, [[COPY16]], 8, [[COPY16]], 0, 0, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: S_CBRANCH_SCC1 %bb.1, implicit killed $scc
; CHECK-NEXT: S_BRANCH %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2.epilogue:
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 0, 0 :: (dereferenceable invariant load (s64) from %ir.out.kernarg.offset550, align 16, addrspace 4)
- ; CHECK-NEXT: [[V_MOV_B32_e32_13:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: GLOBAL_STORE_DWORD_SADDR [[V_MOV_B32_e32_13]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_2]].sub0, [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s32) into %ir.out.load, addrspace 1)
+ ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD_SADDR [[V_MOV_B32_e32_1]], [[V_MFMA_F32_16X16X32_F16_vgprcd_e64_2]].sub0, [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s32) into %ir.out.load, addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
ptr addrspace(1) %out,
<8 x half> %a0, <8 x half> %a1,
diff --git a/llvm/test/CodeGen/AMDGPU/si-fix-sgpr-copies-phi-av-classes.mir b/llvm/test/CodeGen/AMDGPU/si-fix-sgpr-copies-phi-av-classes.mir
new file mode 100644
index 00000000000000..df24206b7058c8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/si-fix-sgpr-copies-phi-av-classes.mir
@@ -0,0 +1,377 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu9.0a -run-pass=si-fix-sgpr-copies -verify-machineinstrs -o - %s | FileCheck %s
+
+# PHIs that are vector from the start are not legalized, whatever the mix of
+# VGPR, AGPR and AV input classes. PHIs with SGPR or subregister inputs are.
+
+---
+name: phi_vgpr_av_operand
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: phi_vgpr_av_operand
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0, $vgpr1, $sgpr0_sgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:av_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+ ; CHECK-NEXT: $vcc = COPY [[COPY2]]
+ ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[COPY1]], [[COPY1]], implicit $exec
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY]], %bb.0, [[V_ADD_U32_e32_]], %bb.1
+ ; CHECK-NEXT: $vgpr0 = COPY [[PHI]]
+ ; CHECK-NEXT: SI_RETURN implicit $vgpr0
+ bb.0:
+ successors: %bb.1, %bb.2
+ liveins: $vgpr0, $vgpr1, $sgpr0_sgpr1
+
+ %0:av_32 = COPY $vgpr0
+ %1:vgpr_32 = COPY $vgpr1
+ %2:sreg_64 = COPY $sgpr0_sgpr1
+ $vcc = COPY %2
+ S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2
+ %3:vgpr_32 = V_ADD_U32_e32 %1, %1, implicit $exec
+
+ bb.2:
+ %4:vgpr_32 = PHI %0, %bb.0, %3, %bb.1
+ $vgpr0 = COPY %4
+ SI_RETURN implicit $vgpr0
+...
+
+---
+name: phi_av_vgpr_operand
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: phi_av_vgpr_operand
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0, $vgpr1, $sgpr0_sgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_32 = COPY $vgpr1
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+ ; CHECK-NEXT: $vcc = COPY [[COPY2]]
+ ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:av_32 = COPY [[COPY1]]
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:av_32 = PHI [[COPY]], %bb.0, [[COPY3]], %bb.1
+ ; CHECK-NEXT: $vgpr0 = COPY [[PHI]]
+ ; CHECK-NEXT: SI_RETURN implicit $vgpr0
+ bb.0:
+ successors: %bb.1, %bb.2
+ liveins: $vgpr0, $vgpr1, $sgpr0_sgpr1
+
+ %0:vgpr_32 = COPY $vgpr0
+ %1:av_32 = COPY $vgpr1
+ %2:sreg_64 = COPY $sgpr0_sgpr1
+ $vcc = COPY %2
+ S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2
+ %3:av_32 = COPY %1
+
+ bb.2:
+ %4:av_32 = PHI %0, %bb.0, %3, %bb.1
+ $vgpr0 = COPY %4
+ SI_RETURN implicit $vgpr0
+...
+
+---
+name: phi_av64_vreg64_operand
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: phi_av64_vreg64_operand
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $sgpr0_sgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_64_align2 = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_64_align2 = COPY $vgpr2_vgpr3
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+ ; CHECK-NEXT: $vcc = COPY [[COPY2]]
+ ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:av_64_align2 = COPY [[COPY1]]
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:av_64_align2 = PHI [[COPY]], %bb.0, [[COPY3]], %bb.1
+ ; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[PHI]]
+ ; CHECK-NEXT: SI_RETURN implicit $vgpr0_vgpr1
+ bb.0:
+ successors: %bb.1, %bb.2
+ liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $sgpr0_sgpr1
+
+ %0:vreg_64_align2 = COPY $vgpr0_vgpr1
+ %1:av_64_align2 = COPY $vgpr2_vgpr3
+ %2:sreg_64 = COPY $sgpr0_sgpr1
+ $vcc = COPY %2
+ S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2
+ %3:av_64_align2 = COPY %1
+
+ bb.2:
+ %4:av_64_align2 = PHI %0, %bb.0, %3, %bb.1
+ $vgpr0_vgpr1 = COPY %4
+ SI_RETURN implicit $vgpr0_vgpr1
+...
+
+# An SGPR operand is copied to a vector register (see #168761).
+---
+name: phi_av_sgpr_operand
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: phi_av_sgpr_operand
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0, $sgpr0_sgpr1, $sgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+ ; CHECK-NEXT: $vcc = COPY [[COPY2]]
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:av_32 = COPY [[COPY]], implicit $exec
+ ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:av_32 = COPY [[COPY1]]
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:av_32 = PHI [[COPY3]], %bb.0, [[COPY4]], %bb.1
+ ; CHECK-NEXT: $vgpr0 = COPY [[PHI]]
+ ; CHECK-NEXT: SI_RETURN implicit $vgpr0
+ bb.0:
+ successors: %bb.1, %bb.2
+ liveins: $vgpr0, $sgpr0_sgpr1, $sgpr2
+
+ %0:sreg_32 = COPY $sgpr2
+ %1:av_32 = COPY $vgpr0
+ %2:sreg_64 = COPY $sgpr0_sgpr1
+ $vcc = COPY %2
+ S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2
+ %3:av_32 = COPY %1
+
+ bb.2:
+ %4:av_32 = PHI %0, %bb.0, %3, %bb.1
+ $vgpr0 = COPY %4
+ SI_RETURN implicit $vgpr0
+...
+
+---
+name: phi_vgpr_agpr_operand
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: phi_vgpr_agpr_operand
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0, $agpr0, $sgpr0_sgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:agpr_32 = COPY $agpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+ ; CHECK-NEXT: $vcc = COPY [[COPY2]]
+ ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[COPY1]], [[COPY1]], implicit $exec
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY]], %bb.0, [[V_ADD_U32_e32_]], %bb.1
+ ; CHECK-NEXT: $vgpr0 = COPY [[PHI]]
+ ; CHECK-NEXT: SI_RETURN implicit $vgpr0
+ bb.0:
+ successors: %bb.1, %bb.2
+ liveins: $vgpr0, $agpr0, $sgpr0_sgpr1
+
+ %0:agpr_32 = COPY $agpr0
+ %1:vgpr_32 = COPY $vgpr0
+ %2:sreg_64 = COPY $sgpr0_sgpr1
+ $vcc = COPY %2
+ S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2
+ %3:vgpr_32 = V_ADD_U32_e32 %1, %1, implicit $exec
+
+ bb.2:
+ %4:vgpr_32 = PHI %0, %bb.0, %3, %bb.1
+ $vgpr0 = COPY %4
+ SI_RETURN implicit $vgpr0
+...
+
+# A subregister operand is copied.
+---
+name: phi_vgpr_av64_subreg_operand
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: phi_vgpr_av64_subreg_operand
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0_sgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:av_64_align2 = COPY $vgpr0_vgpr1
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+ ; CHECK-NEXT: $vcc = COPY [[COPY2]]
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub1, implicit $exec
+ ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[COPY1]], [[COPY1]], implicit $exec
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:vgpr_32 = PHI [[COPY3]], %bb.0, [[V_ADD_U32_e32_]], %bb.1
+ ; CHECK-NEXT: $vgpr0 = COPY [[PHI]]
+ ; CHECK-NEXT: SI_RETURN implicit $vgpr0
+ bb.0:
+ successors: %bb.1, %bb.2
+ liveins: $vgpr0_vgpr1, $vgpr2, $sgpr0_sgpr1
+
+ %0:av_64_align2 = COPY $vgpr0_vgpr1
+ %1:vgpr_32 = COPY $vgpr2
+ %2:sreg_64 = COPY $sgpr0_sgpr1
+ $vcc = COPY %2
+ S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2
+ %3:vgpr_32 = V_ADD_U32_e32 %1, %1, implicit $exec
+
+ bb.2:
+ %4:vgpr_32 = PHI %0.sub1, %bb.0, %3, %bb.1
+ $vgpr0 = COPY %4
+ SI_RETURN implicit $vgpr0
+...
+
+---
+name: phi_agpr_av_operand
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: phi_agpr_av_operand
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0, $agpr0, $sgpr0_sgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:av_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:agpr_32 = COPY $agpr0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+ ; CHECK-NEXT: $vcc = COPY [[COPY2]]
+ ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:agpr_32 = COPY [[COPY1]]
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:agpr_32 = PHI [[COPY]], %bb.0, [[COPY3]], %bb.1
+ ; CHECK-NEXT: $agpr0 = COPY [[PHI]]
+ ; CHECK-NEXT: SI_RETURN implicit $agpr0
+ bb.0:
+ successors: %bb.1, %bb.2
+ liveins: $vgpr0, $agpr0, $sgpr0_sgpr1
+
+ %0:av_32 = COPY $vgpr0
+ %1:agpr_32 = COPY $agpr0
+ %2:sreg_64 = COPY $sgpr0_sgpr1
+ $vcc = COPY %2
+ S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2
+ %3:agpr_32 = COPY %1
+
+ bb.2:
+ %4:agpr_32 = PHI %0, %bb.0, %3, %bb.1
+ $agpr0 = COPY %4
+ SI_RETURN implicit $agpr0
+...
+
+---
+name: phi_av_agpr_operand
+tracksRegLiveness: true
+body: |
+ ; CHECK-LABEL: name: phi_av_agpr_operand
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: liveins: $vgpr0, $agpr0, $sgpr0_sgpr1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:agpr_32 = COPY $agpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:av_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+ ; CHECK-NEXT: $vcc = COPY [[COPY2]]
+ ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:av_32 = COPY [[COPY1]]
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:av_32 = PHI [[COPY]], %bb.0, [[COPY3]], %bb.1
+ ; CHECK-NEXT: $vgpr0 = COPY [[PHI]]
+ ; CHECK-NEXT: SI_RETURN implicit $vgpr0
+ bb.0:
+ successors: %bb.1, %bb.2
+ liveins: $vgpr0, $agpr0, $sgpr0_sgpr1
+
+ %0:agpr_32 = COPY $agpr0
+ %1:av_32 = COPY $vgpr0
+ %2:sreg_64 = COPY $sgpr0_sgpr1
+ $vcc = COPY %2
+ S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2
+ %3:av_32 = COPY %1
+
+ bb.2:
+ %4:av_32 = PHI %0, %bb.0, %3, %bb.1
+ $vgpr0 = COPY %4
+ SI_RETURN implicit $vgpr0
+...
More information about the llvm-commits
mailing list