[clang] [llvm] [clang-tools-extra] [MachineLICM][AArch64] Hoist COPY instructions with other uses in the loop (PR #71403)
via cfe-commits
cfe-commits at lists.llvm.org
Thu Nov 16 06:16:10 PST 2023
https://github.com/Rin18 updated https://github.com/llvm/llvm-project/pull/71403
>From 475cc904b9d21225909fc01a4ef257c091c81d9f Mon Sep 17 00:00:00 2001
From: Rin Dobrescu <rin.dobrescu at arm.com>
Date: Tue, 7 Nov 2023 11:46:53 +0000
Subject: [PATCH 1/3] Squashed commit of the following:
commit fef5786ca1d78288c3f442057bfeb148788417b2
Merge: 77daaf1c4b40 aadaa1cc32b8
Author: Rin Dobrescu <rin.dobrescu at arm.com>
Date: Tue Nov 7 10:27:03 2023 +0000
Merge branch 'hoist_dups' of https://github.com/Rin18/llvm-project-fork into hoist_dups
commit 77daaf1c4b4009bc95803e8992291f64c9cf8b33
Author: Rin Dobrescu <rin.dobrescu at arm.com>
Date: Mon Nov 6 15:52:27 2023 +0000
Remove unneeded line
commit aa469a15af8aeaca8d474ee7ab5b59ff20aec28c
Author: Rin Dobrescu <rin.dobrescu at arm.com>
Date: Mon Nov 6 15:08:11 2023 +0000
Run clang-format on patch
commit 0d43d36e939e702e73b5927b4ae1c713ec0edec9
Author: Rin Dobrescu <rin.dobrescu at arm.com>
Date: Mon Nov 6 12:20:21 2023 +0000
[MachineLICM][AArch64] Hoist COPY instructions with other uses in the loop
commit aadaa1cc32b88e682f0b9148683d3f6142e73f4e
Author: Rin Dobrescu <rin.dobrescu at arm.com>
Date: Mon Nov 6 15:52:27 2023 +0000
Remove unneeded line
commit 27c7354c1bcc5aff81fed7623ff56d93af4201da
Author: Rin Dobrescu <rin.dobrescu at arm.com>
Date: Mon Nov 6 15:08:11 2023 +0000
Run clang-format on patch
commit 34f9b474e3e10f670b6bb0359f2b81ff8275d5a1
Author: Rin Dobrescu <rin.dobrescu at arm.com>
Date: Mon Nov 6 12:20:21 2023 +0000
[MachineLICM][AArch64] Hoist COPY instructions with other uses in the loop
---
llvm/lib/CodeGen/MachineLICM.cpp | 9 +
llvm/test/CodeGen/AArch64/tbl-loops.ll | 126 +-
.../AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll | 60 +-
.../CodeGen/AMDGPU/flat_atomics_i32_system.ll | 1688 +++++++-------
.../CodeGen/AMDGPU/flat_atomics_i64_system.ll | 2028 ++++++++---------
.../CodeGen/AMDGPU/fp64-atomics-gfx90a.ll | 56 +-
.../AMDGPU/global-load-saddr-to-vaddr.ll | 15 +-
.../AMDGPU/global_atomics_i32_system.ll | 700 +++---
.../AMDGPU/global_atomics_i64_system.ll | 1614 +++++++------
llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll | 220 +-
.../CodeGen/AMDGPU/memcpy-crash-issue63986.ll | 24 +-
.../CodeGen/AMDGPU/optimize-negated-cond.ll | 3 +-
.../AMDGPU/promote-constOffset-to-imm.ll | 6 +-
.../CodeGen/AMDGPU/remat-fp64-constants.ll | 2 +-
llvm/test/CodeGen/AMDGPU/sdiv64.ll | 2 +-
llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll | 26 +-
llvm/test/CodeGen/AMDGPU/srem64.ll | 2 +-
llvm/test/CodeGen/AMDGPU/udiv64.ll | 2 +-
llvm/test/CodeGen/AMDGPU/urem64.ll | 2 +-
.../CodeGen/X86/2007-01-13-StackPtrIndex.ll | 45 +-
.../CodeGen/X86/2009-02-26-MachineLICMBug.ll | 2 +-
llvm/test/CodeGen/X86/avx512-i1test.ll | 6 +-
llvm/test/CodeGen/X86/pr38795.ll | 1 -
llvm/test/CodeGen/X86/pr63108.ll | 8 +-
.../X86/vector-shift-by-select-loop.ll | 138 +-
25 files changed, 3240 insertions(+), 3545 deletions(-)
diff --git a/llvm/lib/CodeGen/MachineLICM.cpp b/llvm/lib/CodeGen/MachineLICM.cpp
index e29f28ecaea0dce..b32a39d239a56f3 100644
--- a/llvm/lib/CodeGen/MachineLICM.cpp
+++ b/llvm/lib/CodeGen/MachineLICM.cpp
@@ -1249,6 +1249,15 @@ bool MachineLICMBase::IsProfitableToHoist(MachineInstr &MI,
return false;
}
+ // If we have a COPY with other uses in the loop, hoist to allow the users to
+ // also be hoisted.
+ if (MI.isCopy() && IsLoopInvariantInst(MI, CurLoop) &&
+ MI.getOperand(0).isReg() && MI.getOperand(0).getReg().isVirtual() &&
+ MI.getOperand(1).isReg() && MI.getOperand(1).getReg().isVirtual() &&
+ any_of(MRI->use_nodbg_instructions(MI.getOperand(0).getReg()),
+ [&](MachineInstr &UseMI) { return CurLoop->contains(&UseMI); }))
+ return true;
+
// High register pressure situation, only hoist if the instruction is going
// to be remat'ed.
if (!isTriviallyReMaterializable(MI) &&
diff --git a/llvm/test/CodeGen/AArch64/tbl-loops.ll b/llvm/test/CodeGen/AArch64/tbl-loops.ll
index b63d540fb8e0291..365fe03ab0b0844 100644
--- a/llvm/test/CodeGen/AArch64/tbl-loops.ll
+++ b/llvm/test/CodeGen/AArch64/tbl-loops.ll
@@ -52,19 +52,19 @@ define void @loop1(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n
; CHECK-NEXT: b.eq .LBB0_8
; CHECK-NEXT: .LBB0_6: // %for.body.preheader1
; CHECK-NEXT: movi d0, #0000000000000000
-; CHECK-NEXT: sub w10, w2, w10
; CHECK-NEXT: mov w11, #1132396544 // =0x437f0000
+; CHECK-NEXT: sub w10, w2, w10
+; CHECK-NEXT: fmov s1, w11
; CHECK-NEXT: .LBB0_7: // %for.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: fmov s2, w11
-; CHECK-NEXT: ldr s1, [x8], #4
-; CHECK-NEXT: fcmp s1, s2
-; CHECK-NEXT: fcsel s2, s2, s1, gt
-; CHECK-NEXT: fcmp s1, #0.0
-; CHECK-NEXT: fcsel s1, s0, s2, mi
+; CHECK-NEXT: ldr s2, [x8], #4
+; CHECK-NEXT: fcmp s2, s1
+; CHECK-NEXT: fcsel s3, s1, s2, gt
+; CHECK-NEXT: fcmp s2, #0.0
+; CHECK-NEXT: fcsel s2, s0, s3, mi
; CHECK-NEXT: subs w10, w10, #1
-; CHECK-NEXT: fcvtzs w12, s1
-; CHECK-NEXT: strb w12, [x9], #1
+; CHECK-NEXT: fcvtzs w11, s2
+; CHECK-NEXT: strb w11, [x9], #1
; CHECK-NEXT: b.ne .LBB0_7
; CHECK-NEXT: .LBB0_8: // %for.cond.cleanup
; CHECK-NEXT: ret
@@ -165,25 +165,25 @@ define void @loop2(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n
; CHECK-NEXT: mov x9, x0
; CHECK-NEXT: .LBB1_5: // %for.body.preheader1
; CHECK-NEXT: movi d0, #0000000000000000
-; CHECK-NEXT: sub w10, w2, w10
; CHECK-NEXT: mov w11, #1132396544 // =0x437f0000
+; CHECK-NEXT: sub w10, w2, w10
+; CHECK-NEXT: fmov s1, w11
; CHECK-NEXT: .LBB1_6: // %for.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldp s1, s3, [x8], #8
-; CHECK-NEXT: fmov s2, w11
-; CHECK-NEXT: fcmp s1, s2
-; CHECK-NEXT: fcsel s4, s2, s1, gt
-; CHECK-NEXT: fcmp s1, #0.0
-; CHECK-NEXT: fcsel s1, s0, s4, mi
-; CHECK-NEXT: fcmp s3, s2
-; CHECK-NEXT: fcsel s2, s2, s3, gt
+; CHECK-NEXT: ldp s2, s3, [x8], #8
+; CHECK-NEXT: fcmp s2, s1
+; CHECK-NEXT: fcsel s4, s1, s2, gt
+; CHECK-NEXT: fcmp s2, #0.0
+; CHECK-NEXT: fcsel s2, s0, s4, mi
+; CHECK-NEXT: fcmp s3, s1
+; CHECK-NEXT: fcsel s4, s1, s3, gt
; CHECK-NEXT: fcmp s3, #0.0
-; CHECK-NEXT: fcvtzs w12, s1
-; CHECK-NEXT: fcsel s2, s0, s2, mi
+; CHECK-NEXT: fcvtzs w11, s2
+; CHECK-NEXT: fcsel s3, s0, s4, mi
; CHECK-NEXT: subs w10, w10, #1
-; CHECK-NEXT: strb w12, [x9]
-; CHECK-NEXT: fcvtzs w13, s2
-; CHECK-NEXT: strb w13, [x9, #1]
+; CHECK-NEXT: strb w11, [x9]
+; CHECK-NEXT: fcvtzs w12, s3
+; CHECK-NEXT: strb w12, [x9, #1]
; CHECK-NEXT: add x9, x9, #2
; CHECK-NEXT: b.ne .LBB1_6
; CHECK-NEXT: .LBB1_7: // %for.cond.cleanup
@@ -380,33 +380,33 @@ define void @loop3(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n
; CHECK-NEXT: mov x9, x0
; CHECK-NEXT: .LBB2_7: // %for.body.preheader1
; CHECK-NEXT: movi d0, #0000000000000000
-; CHECK-NEXT: sub w10, w2, w10
; CHECK-NEXT: mov w11, #1132396544 // =0x437f0000
+; CHECK-NEXT: sub w10, w2, w10
+; CHECK-NEXT: fmov s1, w11
; CHECK-NEXT: .LBB2_8: // %for.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldp s1, s3, [x8]
-; CHECK-NEXT: fmov s2, w11
-; CHECK-NEXT: fcmp s1, s2
-; CHECK-NEXT: fcsel s4, s2, s1, gt
-; CHECK-NEXT: fcmp s1, #0.0
-; CHECK-NEXT: fcsel s1, s0, s4, mi
-; CHECK-NEXT: fcmp s3, s2
-; CHECK-NEXT: fcsel s4, s2, s3, gt
+; CHECK-NEXT: ldp s2, s3, [x8]
+; CHECK-NEXT: fcmp s2, s1
+; CHECK-NEXT: fcsel s4, s1, s2, gt
+; CHECK-NEXT: fcmp s2, #0.0
+; CHECK-NEXT: fcsel s2, s0, s4, mi
+; CHECK-NEXT: fcmp s3, s1
+; CHECK-NEXT: fcsel s4, s1, s3, gt
; CHECK-NEXT: fcmp s3, #0.0
; CHECK-NEXT: ldr s3, [x8, #8]
-; CHECK-NEXT: fcvtzs w12, s1
+; CHECK-NEXT: fcvtzs w11, s2
; CHECK-NEXT: add x8, x8, #12
; CHECK-NEXT: fcsel s4, s0, s4, mi
-; CHECK-NEXT: fcmp s3, s2
-; CHECK-NEXT: strb w12, [x9]
-; CHECK-NEXT: fcsel s2, s2, s3, gt
+; CHECK-NEXT: fcmp s3, s1
+; CHECK-NEXT: strb w11, [x9]
+; CHECK-NEXT: fcsel s5, s1, s3, gt
; CHECK-NEXT: fcmp s3, #0.0
-; CHECK-NEXT: fcvtzs w13, s4
-; CHECK-NEXT: fcsel s2, s0, s2, mi
+; CHECK-NEXT: fcvtzs w12, s4
+; CHECK-NEXT: fcsel s3, s0, s5, mi
; CHECK-NEXT: subs w10, w10, #1
-; CHECK-NEXT: strb w13, [x9, #1]
-; CHECK-NEXT: fcvtzs w14, s2
-; CHECK-NEXT: strb w14, [x9, #2]
+; CHECK-NEXT: strb w12, [x9, #1]
+; CHECK-NEXT: fcvtzs w13, s3
+; CHECK-NEXT: strb w13, [x9, #2]
; CHECK-NEXT: add x9, x9, #3
; CHECK-NEXT: b.ne .LBB2_8
; CHECK-NEXT: .LBB2_9: // %for.cond.cleanup
@@ -549,39 +549,39 @@ define void @loop4(ptr noalias nocapture noundef writeonly %dst, ptr nocapture n
; CHECK-NEXT: mov x9, x0
; CHECK-NEXT: .LBB3_5: // %for.body.preheader1
; CHECK-NEXT: movi d0, #0000000000000000
-; CHECK-NEXT: sub w10, w2, w10
; CHECK-NEXT: mov w11, #1132396544 // =0x437f0000
+; CHECK-NEXT: sub w10, w2, w10
+; CHECK-NEXT: fmov s1, w11
; CHECK-NEXT: .LBB3_6: // %for.body
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldp s1, s3, [x8]
-; CHECK-NEXT: fmov s2, w11
-; CHECK-NEXT: fcmp s1, s2
-; CHECK-NEXT: fcsel s4, s2, s1, gt
-; CHECK-NEXT: fcmp s1, #0.0
-; CHECK-NEXT: fcsel s1, s0, s4, mi
-; CHECK-NEXT: fcmp s3, s2
-; CHECK-NEXT: fcsel s4, s2, s3, gt
+; CHECK-NEXT: ldp s2, s3, [x8]
+; CHECK-NEXT: fcmp s2, s1
+; CHECK-NEXT: fcsel s4, s1, s2, gt
+; CHECK-NEXT: fcmp s2, #0.0
+; CHECK-NEXT: fcsel s2, s0, s4, mi
+; CHECK-NEXT: fcmp s3, s1
+; CHECK-NEXT: fcsel s4, s1, s3, gt
; CHECK-NEXT: fcmp s3, #0.0
; CHECK-NEXT: ldp s3, s5, [x8, #8]
-; CHECK-NEXT: fcvtzs w12, s1
+; CHECK-NEXT: fcvtzs w11, s2
; CHECK-NEXT: add x8, x8, #16
; CHECK-NEXT: fcsel s4, s0, s4, mi
-; CHECK-NEXT: fcmp s3, s2
-; CHECK-NEXT: strb w12, [x9]
-; CHECK-NEXT: fcsel s6, s2, s3, gt
+; CHECK-NEXT: fcmp s3, s1
+; CHECK-NEXT: strb w11, [x9]
+; CHECK-NEXT: fcsel s6, s1, s3, gt
; CHECK-NEXT: fcmp s3, #0.0
-; CHECK-NEXT: fcvtzs w13, s4
+; CHECK-NEXT: fcvtzs w12, s4
; CHECK-NEXT: fcsel s3, s0, s6, mi
-; CHECK-NEXT: fcmp s5, s2
-; CHECK-NEXT: strb w13, [x9, #1]
-; CHECK-NEXT: fcsel s2, s2, s5, gt
+; CHECK-NEXT: fcmp s5, s1
+; CHECK-NEXT: strb w12, [x9, #1]
+; CHECK-NEXT: fcsel s6, s1, s5, gt
; CHECK-NEXT: fcmp s5, #0.0
-; CHECK-NEXT: fcvtzs w14, s3
-; CHECK-NEXT: fcsel s2, s0, s2, mi
+; CHECK-NEXT: fcvtzs w13, s3
+; CHECK-NEXT: fcsel s5, s0, s6, mi
; CHECK-NEXT: subs w10, w10, #1
-; CHECK-NEXT: strb w14, [x9, #2]
-; CHECK-NEXT: fcvtzs w15, s2
-; CHECK-NEXT: strb w15, [x9, #3]
+; CHECK-NEXT: strb w13, [x9, #2]
+; CHECK-NEXT: fcvtzs w14, s5
+; CHECK-NEXT: strb w14, [x9, #3]
; CHECK-NEXT: add x9, x9, #4
; CHECK-NEXT: b.ne .LBB3_6
; CHECK-NEXT: .LBB3_7: // %for.cond.cleanup
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
index c6ea046f95a9199..53b2336180c6617 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp64-atomics-gfx90a.ll
@@ -1447,15 +1447,14 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat(ptr %ptr) #1 {
; GFX90A-LABEL: flat_atomic_fadd_f64_noret_pat:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX90A-NEXT: s_mov_b64 s[2:3], 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; GFX90A-NEXT: s_mov_b64 s[0:1], 0
; GFX90A-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -1463,9 +1462,9 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat(ptr %ptr) #1 {
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX90A-NEXT: s_cbranch_execnz .LBB50_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_endpgm
@@ -1522,15 +1521,14 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_system(ptr %ptr) #1 {
; GFX90A-LABEL: flat_atomic_fadd_f64_noret_pat_system:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX90A-NEXT: s_mov_b64 s[2:3], 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; GFX90A-NEXT: s_mov_b64 s[0:1], 0
; GFX90A-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -1539,9 +1537,9 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_system(ptr %ptr) #1 {
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX90A-NEXT: s_cbranch_execnz .LBB52_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_endpgm
@@ -1724,23 +1722,22 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_agent_safe(ptr %ptr) {
; GFX90A-LABEL: flat_atomic_fadd_f64_noret_pat_agent_safe:
; GFX90A: ; %bb.0: ; %main_body
; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX90A-NEXT: s_mov_b64 s[2:3], 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; GFX90A-NEXT: s_mov_b64 s[0:1], 0
; GFX90A-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX90A-NEXT: s_cbranch_execnz .LBB58_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_endpgm
@@ -1957,22 +1954,21 @@ main_body:
define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrspace(3) %ptr) #4 {
; GFX90A-LABEL: local_atomic_fadd_f64_noret_pat_flush_safe:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dword s2, s[0:1], 0x24
-; GFX90A-NEXT: s_mov_b64 s[0:1], 0
+; GFX90A-NEXT: s_load_dword s0, s[0:1], 0x24
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v0, s2
-; GFX90A-NEXT: ds_read_b64 v[0:1], v0
+; GFX90A-NEXT: v_mov_b32_e32 v2, s0
+; GFX90A-NEXT: ds_read_b64 v[0:1], v2
+; GFX90A-NEXT: s_mov_b64 s[0:1], 0
; GFX90A-NEXT: .LBB67_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_add_f64 v[2:3], v[0:1], 4.0
-; GFX90A-NEXT: v_mov_b32_e32 v4, s2
+; GFX90A-NEXT: v_add_f64 v[4:5], v[0:1], 4.0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: ds_cmpst_rtn_b64 v[2:3], v4, v[0:1], v[2:3]
+; GFX90A-NEXT: ds_cmpst_rtn_b64 v[4:5], v2, v[0:1], v[4:5]
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[0:1]
+; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[0:1]
; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX90A-NEXT: s_cbranch_execnz .LBB67_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1985,17 +1981,17 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrsp
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v0, s2
; GFX940-NEXT: ds_read_b64 v[0:1], v0
+; GFX940-NEXT: v_mov_b32_e32 v2, s2
; GFX940-NEXT: .LBB67_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
-; GFX940-NEXT: v_add_f64 v[2:3], v[0:1], 4.0
-; GFX940-NEXT: v_mov_b32_e32 v4, s2
+; GFX940-NEXT: v_add_f64 v[4:5], v[0:1], 4.0
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
-; GFX940-NEXT: ds_cmpst_rtn_b64 v[2:3], v4, v[0:1], v[2:3]
+; GFX940-NEXT: ds_cmpst_rtn_b64 v[4:5], v2, v[0:1], v[4:5]
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
-; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[0:1]
+; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[0:1]
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX940-NEXT: v_mov_b64_e32 v[0:1], v[2:3]
+; GFX940-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_cbranch_execnz .LBB67_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/flat_atomics_i32_system.ll b/llvm/test/CodeGen/AMDGPU/flat_atomics_i32_system.ll
index 6bdeeddc951ff18..c4a88ebf4897294 100644
--- a/llvm/test/CodeGen/AMDGPU/flat_atomics_i32_system.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat_atomics_i32_system.ll
@@ -1819,22 +1819,20 @@ define amdgpu_gfx void @flat_atomic_nand_i32_noret_scalar(ptr inreg %ptr, i32 in
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB44_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: v_not_b32_e32 v0, v0
+; GCN1-NEXT: v_and_b32_e32 v2, s6, v3
+; GCN1-NEXT: v_not_b32_e32 v2, v2
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB44_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1846,22 +1844,20 @@ define amdgpu_gfx void @flat_atomic_nand_i32_noret_scalar(ptr inreg %ptr, i32 in
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB44_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: v_not_b32_e32 v0, v0
+; GCN2-NEXT: v_and_b32_e32 v2, s6, v3
+; GCN2-NEXT: v_not_b32_e32 v2, v2
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB44_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1873,22 +1869,20 @@ define amdgpu_gfx void @flat_atomic_nand_i32_noret_scalar(ptr inreg %ptr, i32 in
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1]
+; GCN3-NEXT: flat_load_dword v3, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB44_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_not_b32_e32 v0, v0
+; GCN3-NEXT: v_and_b32_e32 v2, s6, v3
+; GCN3-NEXT: v_not_b32_e32 v2, v2
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB44_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1906,26 +1900,24 @@ define amdgpu_gfx void @flat_atomic_nand_i32_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB45_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: v_not_b32_e32 v0, v0
+; GCN1-NEXT: v_and_b32_e32 v2, s6, v3
+; GCN1-NEXT: v_not_b32_e32 v2, v2
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB45_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i32_noret_offset_scalar:
@@ -1935,26 +1927,24 @@ define amdgpu_gfx void @flat_atomic_nand_i32_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB45_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: v_not_b32_e32 v0, v0
+; GCN2-NEXT: v_and_b32_e32 v2, s6, v3
+; GCN2-NEXT: v_not_b32_e32 v2, v2
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB45_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i32_noret_offset_scalar:
@@ -1962,22 +1952,20 @@ define amdgpu_gfx void @flat_atomic_nand_i32_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1] offset:16
+; GCN3-NEXT: flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB45_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_not_b32_e32 v0, v0
+; GCN3-NEXT: v_and_b32_e32 v2, s6, v3
+; GCN3-NEXT: v_not_b32_e32 v2, v2
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB45_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1995,20 +1983,20 @@ define amdgpu_gfx i32 @flat_atomic_nand_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: flat_load_dword v0, v[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v1, s4
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v2, s5
; GCN1-NEXT: .LBB46_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: v_not_b32_e32 v0, v0
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_and_b32_e32 v0, s6, v4
+; GCN1-NEXT: v_not_b32_e32 v3, v0
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB46_1
@@ -2022,20 +2010,20 @@ define amdgpu_gfx i32 @flat_atomic_nand_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: flat_load_dword v0, v[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v1, s4
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v2, s5
; GCN2-NEXT: .LBB46_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: v_not_b32_e32 v0, v0
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_and_b32_e32 v0, s6, v4
+; GCN2-NEXT: v_not_b32_e32 v3, v0
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB46_1
@@ -2049,20 +2037,20 @@ define amdgpu_gfx i32 @flat_atomic_nand_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB46_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_not_b32_e32 v0, v0
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_and_b32_e32 v0, s6, v4
+; GCN3-NEXT: v_not_b32_e32 v3, v0
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB46_1
@@ -2079,28 +2067,26 @@ define amdgpu_gfx i32 @flat_atomic_nand_i32_ret_offset_scalar(ptr inreg %out, i3
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 16
; GCN1-NEXT: s_addc_u32 s35, s5, 0
-; GCN1-NEXT: v_mov_b32_e32 v0, s34
-; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s34
+; GCN1-NEXT: v_mov_b32_e32 v2, s35
+; GCN1-NEXT: flat_load_dword v0, v[1:2]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB47_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: v_not_b32_e32 v0, v0
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_and_b32_e32 v0, s6, v4
+; GCN1-NEXT: v_not_b32_e32 v3, v0
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB47_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i32_ret_offset_scalar:
@@ -2108,28 +2094,26 @@ define amdgpu_gfx i32 @flat_atomic_nand_i32_ret_offset_scalar(ptr inreg %out, i3
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 16
; GCN2-NEXT: s_addc_u32 s35, s5, 0
-; GCN2-NEXT: v_mov_b32_e32 v0, s34
-; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s34
+; GCN2-NEXT: v_mov_b32_e32 v2, s35
+; GCN2-NEXT: flat_load_dword v0, v[1:2]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB47_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: v_not_b32_e32 v0, v0
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_and_b32_e32 v0, s6, v4
+; GCN2-NEXT: v_not_b32_e32 v3, v0
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB47_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i32_ret_offset_scalar:
@@ -2138,20 +2122,20 @@ define amdgpu_gfx i32 @flat_atomic_nand_i32_ret_offset_scalar(ptr inreg %out, i3
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1] offset:16
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB47_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_not_b32_e32 v0, v0
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_and_b32_e32 v0, s6, v4
+; GCN3-NEXT: v_not_b32_e32 v3, v0
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB47_1
@@ -3067,21 +3051,19 @@ define amdgpu_gfx void @flat_atomic_max_i32_noret_scalar(ptr inreg %ptr, i32 inr
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB68_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_max_i32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
+; GCN1-NEXT: v_max_i32_e32 v2, s6, v3
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB68_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3093,21 +3075,19 @@ define amdgpu_gfx void @flat_atomic_max_i32_noret_scalar(ptr inreg %ptr, i32 inr
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB68_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_max_i32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
+; GCN2-NEXT: v_max_i32_e32 v2, s6, v3
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB68_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3119,21 +3099,19 @@ define amdgpu_gfx void @flat_atomic_max_i32_noret_scalar(ptr inreg %ptr, i32 inr
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1]
+; GCN3-NEXT: flat_load_dword v3, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB68_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_max_i32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
+; GCN3-NEXT: v_max_i32_e32 v2, s6, v3
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB68_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3151,25 +3129,23 @@ define amdgpu_gfx void @flat_atomic_max_i32_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB69_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_max_i32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
+; GCN1-NEXT: v_max_i32_e32 v2, s6, v3
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB69_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i32_noret_offset_scalar:
@@ -3179,25 +3155,23 @@ define amdgpu_gfx void @flat_atomic_max_i32_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB69_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_max_i32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
+; GCN2-NEXT: v_max_i32_e32 v2, s6, v3
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB69_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i32_noret_offset_scalar:
@@ -3205,21 +3179,19 @@ define amdgpu_gfx void @flat_atomic_max_i32_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1] offset:16
+; GCN3-NEXT: flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB69_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_max_i32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
+; GCN3-NEXT: v_max_i32_e32 v2, s6, v3
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB69_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3237,19 +3209,19 @@ define amdgpu_gfx i32 @flat_atomic_max_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: flat_load_dword v0, v[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v1, s4
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v2, s5
; GCN1-NEXT: .LBB70_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: v_max_i32_e32 v0, s6, v1
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_max_i32_e32 v3, s6, v4
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB70_1
@@ -3263,19 +3235,19 @@ define amdgpu_gfx i32 @flat_atomic_max_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: flat_load_dword v0, v[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v1, s4
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v2, s5
; GCN2-NEXT: .LBB70_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: v_max_i32_e32 v0, s6, v1
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_max_i32_e32 v3, s6, v4
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB70_1
@@ -3289,19 +3261,19 @@ define amdgpu_gfx i32 @flat_atomic_max_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB70_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_max_i32_e32 v0, s6, v1
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_max_i32_e32 v3, s6, v4
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB70_1
@@ -3318,27 +3290,25 @@ define amdgpu_gfx i32 @flat_atomic_max_i32_ret_offset_scalar(ptr inreg %out, i32
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 16
; GCN1-NEXT: s_addc_u32 s35, s5, 0
-; GCN1-NEXT: v_mov_b32_e32 v0, s34
-; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s34
+; GCN1-NEXT: v_mov_b32_e32 v2, s35
+; GCN1-NEXT: flat_load_dword v0, v[1:2]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB71_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: v_max_i32_e32 v0, s6, v1
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_max_i32_e32 v3, s6, v4
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB71_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i32_ret_offset_scalar:
@@ -3346,27 +3316,25 @@ define amdgpu_gfx i32 @flat_atomic_max_i32_ret_offset_scalar(ptr inreg %out, i32
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 16
; GCN2-NEXT: s_addc_u32 s35, s5, 0
-; GCN2-NEXT: v_mov_b32_e32 v0, s34
-; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s34
+; GCN2-NEXT: v_mov_b32_e32 v2, s35
+; GCN2-NEXT: flat_load_dword v0, v[1:2]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB71_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: v_max_i32_e32 v0, s6, v1
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_max_i32_e32 v3, s6, v4
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB71_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i32_ret_offset_scalar:
@@ -3375,19 +3343,19 @@ define amdgpu_gfx i32 @flat_atomic_max_i32_ret_offset_scalar(ptr inreg %out, i32
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1] offset:16
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB71_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_max_i32_e32 v0, s6, v1
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_max_i32_e32 v3, s6, v4
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB71_1
@@ -3413,22 +3381,20 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr %out, i32 %in, i32 %
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[4:5], 0
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: .LBB72_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT: v_mov_b32_e32 v3, s1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_max_i32_e32 v0, s2, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s0
+; GCN1-NEXT: v_max_i32_e32 v2, s2, v3
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB72_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_endpgm
@@ -3446,22 +3412,20 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr %out, i32 %in, i32 %
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[4:5], 0
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: .LBB72_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT: v_mov_b32_e32 v3, s1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_max_i32_e32 v0, s2, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s0
+; GCN2-NEXT: v_max_i32_e32 v2, s2, v3
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB72_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_endpgm
@@ -3477,22 +3441,20 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr %out, i32 %in, i32 %
; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dword v1, v[0:1] offset:16
-; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: flat_load_dword v3, v[0:1] offset:16
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: .LBB72_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT: v_mov_b32_e32 v3, s1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_max_i32_e32 v0, s2, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s0
+; GCN3-NEXT: v_max_i32_e32 v2, s2, v3
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB72_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_endpgm
@@ -3518,28 +3480,26 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr %out, ptr %out2,
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[6:7], 0
+; GCN1-NEXT: flat_load_dword v2, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: .LBB73_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v3, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s0
-; GCN1-NEXT: v_max_i32_e32 v0, s4, v1
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: v_max_i32_e32 v2, s4, v3
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB73_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN1-NEXT: v_mov_b32_e32 v1, s2
-; GCN1-NEXT: v_mov_b32_e32 v2, s3
-; GCN1-NEXT: flat_store_dword v[1:2], v0
+; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v0, s2
+; GCN1-NEXT: v_mov_b32_e32 v1, s3
+; GCN1-NEXT: flat_store_dword v[0:1], v2
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_max_i32_ret_addr64_offset:
@@ -3556,28 +3516,26 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[6:7], 0
+; GCN2-NEXT: flat_load_dword v2, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: .LBB73_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v3, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s0
-; GCN2-NEXT: v_max_i32_e32 v0, s4, v1
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: v_max_i32_e32 v2, s4, v3
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB73_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN2-NEXT: v_mov_b32_e32 v1, s2
-; GCN2-NEXT: v_mov_b32_e32 v2, s3
-; GCN2-NEXT: flat_store_dword v[1:2], v0
+; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v0, s2
+; GCN2-NEXT: v_mov_b32_e32 v1, s3
+; GCN2-NEXT: flat_store_dword v[0:1], v2
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_max_i32_ret_addr64_offset:
@@ -3592,28 +3550,26 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr %out, ptr %out2,
; GCN3-NEXT: s_addc_u32 s1, s5, s1
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dword v0, v[0:1] offset:16
-; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: flat_load_dword v2, v[0:1] offset:16
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: .LBB73_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v3, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s0
-; GCN3-NEXT: v_max_i32_e32 v0, s2, v1
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
+; GCN3-NEXT: v_max_i32_e32 v2, s2, v3
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB73_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT: v_mov_b32_e32 v1, s6
-; GCN3-NEXT: v_mov_b32_e32 v2, s7
-; GCN3-NEXT: flat_store_dword v[1:2], v0
+; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v0, s6
+; GCN3-NEXT: v_mov_b32_e32 v1, s7
+; GCN3-NEXT: flat_store_dword v[0:1], v2
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr i32, ptr %out, i32 %index
@@ -3635,22 +3591,20 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr %out, i32 %in, i32 %index)
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[4:5], 0
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: .LBB74_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT: v_mov_b32_e32 v3, s1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_max_i32_e32 v0, s2, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s0
+; GCN1-NEXT: v_max_i32_e32 v2, s2, v3
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB74_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_endpgm
@@ -3666,22 +3620,20 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr %out, i32 %in, i32 %index)
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[4:5], 0
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: .LBB74_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT: v_mov_b32_e32 v3, s1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_max_i32_e32 v0, s2, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s0
+; GCN2-NEXT: v_max_i32_e32 v2, s2, v3
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB74_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_endpgm
@@ -3697,22 +3649,20 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr %out, i32 %in, i32 %index)
; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dword v1, v[0:1]
-; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: flat_load_dword v3, v[0:1]
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: .LBB74_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT: v_mov_b32_e32 v3, s1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_max_i32_e32 v0, s2, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s0
+; GCN3-NEXT: v_max_i32_e32 v2, s2, v3
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB74_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_endpgm
@@ -3735,28 +3685,26 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr %out, ptr %out2, i32 %i
; GCN1-NEXT: s_addc_u32 s1, s1, s7
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[6:7], 0
+; GCN1-NEXT: flat_load_dword v2, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: .LBB75_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v3, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s0
-; GCN1-NEXT: v_max_i32_e32 v0, s4, v1
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: v_max_i32_e32 v2, s4, v3
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB75_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN1-NEXT: v_mov_b32_e32 v1, s2
-; GCN1-NEXT: v_mov_b32_e32 v2, s3
-; GCN1-NEXT: flat_store_dword v[1:2], v0
+; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v0, s2
+; GCN1-NEXT: v_mov_b32_e32 v1, s3
+; GCN1-NEXT: flat_store_dword v[0:1], v2
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_max_i32_ret_addr64:
@@ -3771,28 +3719,26 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr %out, ptr %out2, i32 %i
; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[6:7], 0
+; GCN2-NEXT: flat_load_dword v2, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: .LBB75_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v3, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s0
-; GCN2-NEXT: v_max_i32_e32 v0, s4, v1
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: v_max_i32_e32 v2, s4, v3
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB75_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN2-NEXT: v_mov_b32_e32 v1, s2
-; GCN2-NEXT: v_mov_b32_e32 v2, s3
-; GCN2-NEXT: flat_store_dword v[1:2], v0
+; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v0, s2
+; GCN2-NEXT: v_mov_b32_e32 v1, s3
+; GCN2-NEXT: flat_store_dword v[0:1], v2
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_max_i32_ret_addr64:
@@ -3807,28 +3753,26 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr %out, ptr %out2, i32 %i
; GCN3-NEXT: s_addc_u32 s1, s5, s1
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dword v0, v[0:1]
-; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: flat_load_dword v2, v[0:1]
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: .LBB75_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v3, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s0
-; GCN3-NEXT: v_max_i32_e32 v0, s2, v1
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
+; GCN3-NEXT: v_max_i32_e32 v2, s2, v3
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB75_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT: v_mov_b32_e32 v1, s6
-; GCN3-NEXT: v_mov_b32_e32 v2, s7
-; GCN3-NEXT: flat_store_dword v[1:2], v0
+; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v0, s6
+; GCN3-NEXT: v_mov_b32_e32 v1, s7
+; GCN3-NEXT: flat_store_dword v[0:1], v2
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr i32, ptr %out, i32 %index
@@ -4141,21 +4085,19 @@ define amdgpu_gfx void @flat_atomic_umax_i32_noret_scalar(ptr inreg %ptr, i32 in
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB80_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_max_u32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
+; GCN1-NEXT: v_max_u32_e32 v2, s6, v3
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB80_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4167,21 +4109,19 @@ define amdgpu_gfx void @flat_atomic_umax_i32_noret_scalar(ptr inreg %ptr, i32 in
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB80_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_max_u32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
+; GCN2-NEXT: v_max_u32_e32 v2, s6, v3
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB80_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4193,21 +4133,19 @@ define amdgpu_gfx void @flat_atomic_umax_i32_noret_scalar(ptr inreg %ptr, i32 in
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1]
+; GCN3-NEXT: flat_load_dword v3, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB80_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_max_u32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
+; GCN3-NEXT: v_max_u32_e32 v2, s6, v3
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB80_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4225,25 +4163,23 @@ define amdgpu_gfx void @flat_atomic_umax_i32_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB81_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_max_u32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
+; GCN1-NEXT: v_max_u32_e32 v2, s6, v3
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB81_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i32_noret_offset_scalar:
@@ -4253,25 +4189,23 @@ define amdgpu_gfx void @flat_atomic_umax_i32_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB81_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_max_u32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
+; GCN2-NEXT: v_max_u32_e32 v2, s6, v3
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB81_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i32_noret_offset_scalar:
@@ -4279,21 +4213,19 @@ define amdgpu_gfx void @flat_atomic_umax_i32_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1] offset:16
+; GCN3-NEXT: flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB81_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_max_u32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
+; GCN3-NEXT: v_max_u32_e32 v2, s6, v3
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB81_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4311,19 +4243,19 @@ define amdgpu_gfx i32 @flat_atomic_umax_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: flat_load_dword v0, v[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v1, s4
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v2, s5
; GCN1-NEXT: .LBB82_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: v_max_u32_e32 v0, s6, v1
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_max_u32_e32 v3, s6, v4
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB82_1
@@ -4337,19 +4269,19 @@ define amdgpu_gfx i32 @flat_atomic_umax_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: flat_load_dword v0, v[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v1, s4
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v2, s5
; GCN2-NEXT: .LBB82_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: v_max_u32_e32 v0, s6, v1
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_max_u32_e32 v3, s6, v4
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB82_1
@@ -4363,19 +4295,19 @@ define amdgpu_gfx i32 @flat_atomic_umax_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB82_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_max_u32_e32 v0, s6, v1
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_max_u32_e32 v3, s6, v4
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB82_1
@@ -4392,27 +4324,25 @@ define amdgpu_gfx i32 @flat_atomic_umax_i32_ret_offset_scalar(ptr inreg %out, i3
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 16
; GCN1-NEXT: s_addc_u32 s35, s5, 0
-; GCN1-NEXT: v_mov_b32_e32 v0, s34
-; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s34
+; GCN1-NEXT: v_mov_b32_e32 v2, s35
+; GCN1-NEXT: flat_load_dword v0, v[1:2]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB83_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: v_max_u32_e32 v0, s6, v1
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_max_u32_e32 v3, s6, v4
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB83_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i32_ret_offset_scalar:
@@ -4420,27 +4350,25 @@ define amdgpu_gfx i32 @flat_atomic_umax_i32_ret_offset_scalar(ptr inreg %out, i3
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 16
; GCN2-NEXT: s_addc_u32 s35, s5, 0
-; GCN2-NEXT: v_mov_b32_e32 v0, s34
-; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s34
+; GCN2-NEXT: v_mov_b32_e32 v2, s35
+; GCN2-NEXT: flat_load_dword v0, v[1:2]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB83_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: v_max_u32_e32 v0, s6, v1
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_max_u32_e32 v3, s6, v4
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB83_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i32_ret_offset_scalar:
@@ -4449,19 +4377,19 @@ define amdgpu_gfx i32 @flat_atomic_umax_i32_ret_offset_scalar(ptr inreg %out, i3
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1] offset:16
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB83_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_max_u32_e32 v0, s6, v1
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_max_u32_e32 v3, s6, v4
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB83_1
@@ -4487,22 +4415,20 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr %out, i32 %in, i32
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[4:5], 0
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: .LBB84_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT: v_mov_b32_e32 v3, s1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_max_u32_e32 v0, s2, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s0
+; GCN1-NEXT: v_max_u32_e32 v2, s2, v3
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB84_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_endpgm
@@ -4520,22 +4446,20 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr %out, i32 %in, i32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[4:5], 0
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: .LBB84_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT: v_mov_b32_e32 v3, s1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_max_u32_e32 v0, s2, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s0
+; GCN2-NEXT: v_max_u32_e32 v2, s2, v3
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB84_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_endpgm
@@ -4551,22 +4475,20 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr %out, i32 %in, i32
; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dword v1, v[0:1] offset:16
-; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: flat_load_dword v3, v[0:1] offset:16
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: .LBB84_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT: v_mov_b32_e32 v3, s1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_max_u32_e32 v0, s2, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s0
+; GCN3-NEXT: v_max_u32_e32 v2, s2, v3
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB84_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_endpgm
@@ -4592,28 +4514,26 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr %out, ptr %out2
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[6:7], 0
+; GCN1-NEXT: flat_load_dword v2, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: .LBB85_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v3, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s0
-; GCN1-NEXT: v_max_u32_e32 v0, s4, v1
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: v_max_u32_e32 v2, s4, v3
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB85_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN1-NEXT: v_mov_b32_e32 v1, s2
-; GCN1-NEXT: v_mov_b32_e32 v2, s3
-; GCN1-NEXT: flat_store_dword v[1:2], v0
+; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v0, s2
+; GCN1-NEXT: v_mov_b32_e32 v1, s3
+; GCN1-NEXT: flat_store_dword v[0:1], v2
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_umax_i32_ret_addr64_offset:
@@ -4630,28 +4550,26 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr %out, ptr %out2
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[6:7], 0
+; GCN2-NEXT: flat_load_dword v2, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: .LBB85_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v3, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s0
-; GCN2-NEXT: v_max_u32_e32 v0, s4, v1
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: v_max_u32_e32 v2, s4, v3
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB85_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN2-NEXT: v_mov_b32_e32 v1, s2
-; GCN2-NEXT: v_mov_b32_e32 v2, s3
-; GCN2-NEXT: flat_store_dword v[1:2], v0
+; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v0, s2
+; GCN2-NEXT: v_mov_b32_e32 v1, s3
+; GCN2-NEXT: flat_store_dword v[0:1], v2
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_umax_i32_ret_addr64_offset:
@@ -4666,28 +4584,26 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr %out, ptr %out2
; GCN3-NEXT: s_addc_u32 s1, s5, s1
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dword v0, v[0:1] offset:16
-; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: flat_load_dword v2, v[0:1] offset:16
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: .LBB85_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v3, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s0
-; GCN3-NEXT: v_max_u32_e32 v0, s2, v1
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
+; GCN3-NEXT: v_max_u32_e32 v2, s2, v3
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB85_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT: v_mov_b32_e32 v1, s6
-; GCN3-NEXT: v_mov_b32_e32 v2, s7
-; GCN3-NEXT: flat_store_dword v[1:2], v0
+; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v0, s6
+; GCN3-NEXT: v_mov_b32_e32 v1, s7
+; GCN3-NEXT: flat_store_dword v[0:1], v2
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr i32, ptr %out, i32 %index
@@ -4710,28 +4626,26 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr %out, ptr %out2, i32 %
; GCN1-NEXT: s_addc_u32 s1, s1, s7
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[6:7], 0
+; GCN1-NEXT: flat_load_dword v2, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: .LBB86_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v3, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s0
-; GCN1-NEXT: v_max_u32_e32 v0, s4, v1
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: v_max_u32_e32 v2, s4, v3
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB86_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN1-NEXT: v_mov_b32_e32 v1, s2
-; GCN1-NEXT: v_mov_b32_e32 v2, s3
-; GCN1-NEXT: flat_store_dword v[1:2], v0
+; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v0, s2
+; GCN1-NEXT: v_mov_b32_e32 v1, s3
+; GCN1-NEXT: flat_store_dword v[0:1], v2
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_umax_i32_ret_addr64:
@@ -4746,28 +4660,26 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr %out, ptr %out2, i32 %
; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[6:7], 0
+; GCN2-NEXT: flat_load_dword v2, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: .LBB86_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v3, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s0
-; GCN2-NEXT: v_max_u32_e32 v0, s4, v1
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: v_max_u32_e32 v2, s4, v3
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB86_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN2-NEXT: v_mov_b32_e32 v1, s2
-; GCN2-NEXT: v_mov_b32_e32 v2, s3
-; GCN2-NEXT: flat_store_dword v[1:2], v0
+; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v0, s2
+; GCN2-NEXT: v_mov_b32_e32 v1, s3
+; GCN2-NEXT: flat_store_dword v[0:1], v2
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_umax_i32_ret_addr64:
@@ -4782,28 +4694,26 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr %out, ptr %out2, i32 %
; GCN3-NEXT: s_addc_u32 s1, s5, s1
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dword v0, v[0:1]
-; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: flat_load_dword v2, v[0:1]
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: .LBB86_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v3, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s0
-; GCN3-NEXT: v_max_u32_e32 v0, s2, v1
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
+; GCN3-NEXT: v_max_u32_e32 v2, s2, v3
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB86_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT: v_mov_b32_e32 v1, s6
-; GCN3-NEXT: v_mov_b32_e32 v2, s7
-; GCN3-NEXT: flat_store_dword v[1:2], v0
+; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v0, s6
+; GCN3-NEXT: v_mov_b32_e32 v1, s7
+; GCN3-NEXT: flat_store_dword v[0:1], v2
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr i32, ptr %out, i32 %index
@@ -5116,21 +5026,19 @@ define amdgpu_gfx void @flat_atomic_umin_i32_noret_scalar(ptr inreg %ptr, i32 in
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB91_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_min_u32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
+; GCN1-NEXT: v_min_u32_e32 v2, s6, v3
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB91_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -5142,21 +5050,19 @@ define amdgpu_gfx void @flat_atomic_umin_i32_noret_scalar(ptr inreg %ptr, i32 in
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB91_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_min_u32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
+; GCN2-NEXT: v_min_u32_e32 v2, s6, v3
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB91_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -5168,21 +5074,19 @@ define amdgpu_gfx void @flat_atomic_umin_i32_noret_scalar(ptr inreg %ptr, i32 in
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1]
+; GCN3-NEXT: flat_load_dword v3, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB91_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_min_u32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
+; GCN3-NEXT: v_min_u32_e32 v2, s6, v3
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB91_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -5200,25 +5104,23 @@ define amdgpu_gfx void @flat_atomic_umin_i32_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB92_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_min_u32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
+; GCN1-NEXT: v_min_u32_e32 v2, s6, v3
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB92_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i32_noret_offset_scalar:
@@ -5228,25 +5130,23 @@ define amdgpu_gfx void @flat_atomic_umin_i32_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB92_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_min_u32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
+; GCN2-NEXT: v_min_u32_e32 v2, s6, v3
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB92_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i32_noret_offset_scalar:
@@ -5254,21 +5154,19 @@ define amdgpu_gfx void @flat_atomic_umin_i32_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1] offset:16
+; GCN3-NEXT: flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB92_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_min_u32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
+; GCN3-NEXT: v_min_u32_e32 v2, s6, v3
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB92_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -5286,19 +5184,19 @@ define amdgpu_gfx i32 @flat_atomic_umin_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: flat_load_dword v0, v[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v1, s4
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v2, s5
; GCN1-NEXT: .LBB93_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: v_min_u32_e32 v0, s6, v1
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_min_u32_e32 v3, s6, v4
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB93_1
@@ -5312,19 +5210,19 @@ define amdgpu_gfx i32 @flat_atomic_umin_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: flat_load_dword v0, v[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v1, s4
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v2, s5
; GCN2-NEXT: .LBB93_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: v_min_u32_e32 v0, s6, v1
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_min_u32_e32 v3, s6, v4
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB93_1
@@ -5338,19 +5236,19 @@ define amdgpu_gfx i32 @flat_atomic_umin_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB93_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_min_u32_e32 v0, s6, v1
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_min_u32_e32 v3, s6, v4
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB93_1
@@ -5367,27 +5265,25 @@ define amdgpu_gfx i32 @flat_atomic_umin_i32_ret_offset_scalar(ptr inreg %out, i3
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 16
; GCN1-NEXT: s_addc_u32 s35, s5, 0
-; GCN1-NEXT: v_mov_b32_e32 v0, s34
-; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s34
+; GCN1-NEXT: v_mov_b32_e32 v2, s35
+; GCN1-NEXT: flat_load_dword v0, v[1:2]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB94_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: v_min_u32_e32 v0, s6, v1
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_min_u32_e32 v3, s6, v4
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB94_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i32_ret_offset_scalar:
@@ -5395,27 +5291,25 @@ define amdgpu_gfx i32 @flat_atomic_umin_i32_ret_offset_scalar(ptr inreg %out, i3
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 16
; GCN2-NEXT: s_addc_u32 s35, s5, 0
-; GCN2-NEXT: v_mov_b32_e32 v0, s34
-; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s34
+; GCN2-NEXT: v_mov_b32_e32 v2, s35
+; GCN2-NEXT: flat_load_dword v0, v[1:2]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB94_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: v_min_u32_e32 v0, s6, v1
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_min_u32_e32 v3, s6, v4
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB94_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i32_ret_offset_scalar:
@@ -5424,19 +5318,19 @@ define amdgpu_gfx i32 @flat_atomic_umin_i32_ret_offset_scalar(ptr inreg %out, i3
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1] offset:16
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB94_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_min_u32_e32 v0, s6, v1
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_min_u32_e32 v3, s6, v4
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB94_1
@@ -5752,21 +5646,19 @@ define amdgpu_gfx void @flat_atomic_min_i32_noret_scalar(ptr inreg %ptr, i32 inr
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB99_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
+; GCN1-NEXT: v_min_i32_e32 v2, s6, v3
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB99_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -5778,21 +5670,19 @@ define amdgpu_gfx void @flat_atomic_min_i32_noret_scalar(ptr inreg %ptr, i32 inr
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB99_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
+; GCN2-NEXT: v_min_i32_e32 v2, s6, v3
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB99_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -5804,21 +5694,19 @@ define amdgpu_gfx void @flat_atomic_min_i32_noret_scalar(ptr inreg %ptr, i32 inr
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1]
+; GCN3-NEXT: flat_load_dword v3, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB99_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
+; GCN3-NEXT: v_min_i32_e32 v2, s6, v3
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB99_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -5836,25 +5724,23 @@ define amdgpu_gfx void @flat_atomic_min_i32_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB100_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
+; GCN1-NEXT: v_min_i32_e32 v2, s6, v3
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB100_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i32_noret_offset_scalar:
@@ -5864,25 +5750,23 @@ define amdgpu_gfx void @flat_atomic_min_i32_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB100_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
+; GCN2-NEXT: v_min_i32_e32 v2, s6, v3
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB100_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i32_noret_offset_scalar:
@@ -5890,21 +5774,19 @@ define amdgpu_gfx void @flat_atomic_min_i32_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1] offset:16
+; GCN3-NEXT: flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB100_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
+; GCN3-NEXT: v_min_i32_e32 v2, s6, v3
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB100_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -5922,19 +5804,19 @@ define amdgpu_gfx i32 @flat_atomic_min_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: flat_load_dword v0, v[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v1, s4
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v2, s5
; GCN1-NEXT: .LBB101_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: v_min_i32_e32 v0, s6, v1
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_min_i32_e32 v3, s6, v4
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB101_1
@@ -5948,19 +5830,19 @@ define amdgpu_gfx i32 @flat_atomic_min_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: flat_load_dword v0, v[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v1, s4
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v2, s5
; GCN2-NEXT: .LBB101_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: v_min_i32_e32 v0, s6, v1
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_min_i32_e32 v3, s6, v4
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB101_1
@@ -5974,19 +5856,19 @@ define amdgpu_gfx i32 @flat_atomic_min_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB101_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_min_i32_e32 v0, s6, v1
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_min_i32_e32 v3, s6, v4
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB101_1
@@ -6003,27 +5885,25 @@ define amdgpu_gfx i32 @flat_atomic_min_i32_ret_offset_scalar(ptr inreg %out, i32
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 16
; GCN1-NEXT: s_addc_u32 s35, s5, 0
-; GCN1-NEXT: v_mov_b32_e32 v0, s34
-; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s34
+; GCN1-NEXT: v_mov_b32_e32 v2, s35
+; GCN1-NEXT: flat_load_dword v0, v[1:2]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB102_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: v_min_i32_e32 v0, s6, v1
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_min_i32_e32 v3, s6, v4
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB102_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i32_ret_offset_scalar:
@@ -6031,27 +5911,25 @@ define amdgpu_gfx i32 @flat_atomic_min_i32_ret_offset_scalar(ptr inreg %out, i32
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 16
; GCN2-NEXT: s_addc_u32 s35, s5, 0
-; GCN2-NEXT: v_mov_b32_e32 v0, s34
-; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s34
+; GCN2-NEXT: v_mov_b32_e32 v2, s35
+; GCN2-NEXT: flat_load_dword v0, v[1:2]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB102_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: v_min_i32_e32 v0, s6, v1
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_min_i32_e32 v3, s6, v4
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB102_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i32_ret_offset_scalar:
@@ -6060,19 +5938,19 @@ define amdgpu_gfx i32 @flat_atomic_min_i32_ret_offset_scalar(ptr inreg %out, i32
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1] offset:16
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB102_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_min_i32_e32 v0, s6, v1
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_min_i32_e32 v3, s6, v4
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB102_1
@@ -6098,22 +5976,20 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr %out, i32 %in, i32 %
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[4:5], 0
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: .LBB103_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT: v_mov_b32_e32 v3, s1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_min_i32_e32 v0, s2, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s0
+; GCN1-NEXT: v_min_i32_e32 v2, s2, v3
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB103_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_endpgm
@@ -6131,22 +6007,20 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr %out, i32 %in, i32 %
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[4:5], 0
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: .LBB103_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT: v_mov_b32_e32 v3, s1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_min_i32_e32 v0, s2, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s0
+; GCN2-NEXT: v_min_i32_e32 v2, s2, v3
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB103_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_endpgm
@@ -6162,22 +6036,20 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr %out, i32 %in, i32 %
; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dword v1, v[0:1] offset:16
-; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: flat_load_dword v3, v[0:1] offset:16
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: .LBB103_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT: v_mov_b32_e32 v3, s1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_min_i32_e32 v0, s2, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s0
+; GCN3-NEXT: v_min_i32_e32 v2, s2, v3
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB103_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_endpgm
@@ -6203,28 +6075,26 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr %out, ptr %out2,
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[6:7], 0
+; GCN1-NEXT: flat_load_dword v2, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: .LBB104_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v3, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s0
-; GCN1-NEXT: v_min_i32_e32 v0, s4, v1
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: v_min_i32_e32 v2, s4, v3
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB104_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN1-NEXT: v_mov_b32_e32 v1, s2
-; GCN1-NEXT: v_mov_b32_e32 v2, s3
-; GCN1-NEXT: flat_store_dword v[1:2], v0
+; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v0, s2
+; GCN1-NEXT: v_mov_b32_e32 v1, s3
+; GCN1-NEXT: flat_store_dword v[0:1], v2
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_min_i32_ret_addr64_offset:
@@ -6241,28 +6111,26 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[6:7], 0
+; GCN2-NEXT: flat_load_dword v2, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: .LBB104_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v3, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s0
-; GCN2-NEXT: v_min_i32_e32 v0, s4, v1
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: v_min_i32_e32 v2, s4, v3
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB104_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN2-NEXT: v_mov_b32_e32 v1, s2
-; GCN2-NEXT: v_mov_b32_e32 v2, s3
-; GCN2-NEXT: flat_store_dword v[1:2], v0
+; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v0, s2
+; GCN2-NEXT: v_mov_b32_e32 v1, s3
+; GCN2-NEXT: flat_store_dword v[0:1], v2
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_min_i32_ret_addr64_offset:
@@ -6277,28 +6145,26 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr %out, ptr %out2,
; GCN3-NEXT: s_addc_u32 s1, s5, s1
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dword v0, v[0:1] offset:16
-; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: flat_load_dword v2, v[0:1] offset:16
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: .LBB104_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v3, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s0
-; GCN3-NEXT: v_min_i32_e32 v0, s2, v1
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
+; GCN3-NEXT: v_min_i32_e32 v2, s2, v3
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB104_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT: v_mov_b32_e32 v1, s6
-; GCN3-NEXT: v_mov_b32_e32 v2, s7
-; GCN3-NEXT: flat_store_dword v[1:2], v0
+; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v0, s6
+; GCN3-NEXT: v_mov_b32_e32 v1, s7
+; GCN3-NEXT: flat_store_dword v[0:1], v2
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr i32, ptr %out, i32 %index
@@ -6311,26 +6177,24 @@ entry:
define amdgpu_kernel void @atomic_min_i32(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_min_i32:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x9
-; GCN1-NEXT: s_load_dword s4, s[0:1], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9
+; GCN1-NEXT: s_load_dword s2, s[0:1], 0xb
; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v0, s2
-; GCN1-NEXT: v_mov_b32_e32 v1, s3
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v0, s4
+; GCN1-NEXT: v_mov_b32_e32 v1, s5
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: .LBB105_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_min_i32_e32 v0, s4, v1
-; GCN1-NEXT: v_mov_b32_e32 v3, s3
+; GCN1-NEXT: v_min_i32_e32 v2, s2, v3
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB105_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6338,26 +6202,24 @@ define amdgpu_kernel void @atomic_min_i32(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_min_i32:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
-; GCN2-NEXT: s_load_dword s4, s[0:1], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GCN2-NEXT: s_load_dword s2, s[0:1], 0x2c
; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v0, s2
-; GCN2-NEXT: v_mov_b32_e32 v1, s3
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v0, s4
+; GCN2-NEXT: v_mov_b32_e32 v1, s5
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: .LBB105_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_min_i32_e32 v0, s4, v1
-; GCN2-NEXT: v_mov_b32_e32 v3, s3
+; GCN2-NEXT: v_min_i32_e32 v2, s2, v3
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB105_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6365,26 +6227,24 @@ define amdgpu_kernel void @atomic_min_i32(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_min_i32:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
-; GCN3-NEXT: s_load_dword s4, s[0:1], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GCN3-NEXT: s_load_dword s2, s[0:1], 0x2c
; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v0, s2
-; GCN3-NEXT: v_mov_b32_e32 v1, s3
-; GCN3-NEXT: flat_load_dword v1, v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v0, s4
+; GCN3-NEXT: v_mov_b32_e32 v1, s5
+; GCN3-NEXT: flat_load_dword v3, v[0:1]
; GCN3-NEXT: .LBB105_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_min_i32_e32 v0, s4, v1
-; GCN3-NEXT: v_mov_b32_e32 v3, s3
+; GCN3-NEXT: v_min_i32_e32 v2, s2, v3
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB105_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6407,28 +6267,26 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr %out, ptr %out2, i32 %i
; GCN1-NEXT: s_addc_u32 s1, s1, s7
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[6:7], 0
+; GCN1-NEXT: flat_load_dword v2, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: .LBB106_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v3, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s0
-; GCN1-NEXT: v_min_i32_e32 v0, s4, v1
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: v_min_i32_e32 v2, s4, v3
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB106_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN1-NEXT: v_mov_b32_e32 v1, s2
-; GCN1-NEXT: v_mov_b32_e32 v2, s3
-; GCN1-NEXT: flat_store_dword v[1:2], v0
+; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v0, s2
+; GCN1-NEXT: v_mov_b32_e32 v1, s3
+; GCN1-NEXT: flat_store_dword v[0:1], v2
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_min_i32_ret_addr64:
@@ -6443,28 +6301,26 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr %out, ptr %out2, i32 %i
; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[6:7], 0
+; GCN2-NEXT: flat_load_dword v2, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: .LBB106_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v3, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s0
-; GCN2-NEXT: v_min_i32_e32 v0, s4, v1
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: v_min_i32_e32 v2, s4, v3
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB106_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN2-NEXT: v_mov_b32_e32 v1, s2
-; GCN2-NEXT: v_mov_b32_e32 v2, s3
-; GCN2-NEXT: flat_store_dword v[1:2], v0
+; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v0, s2
+; GCN2-NEXT: v_mov_b32_e32 v1, s3
+; GCN2-NEXT: flat_store_dword v[0:1], v2
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_min_i32_ret_addr64:
@@ -6479,28 +6335,26 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr %out, ptr %out2, i32 %i
; GCN3-NEXT: s_addc_u32 s1, s5, s1
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dword v0, v[0:1]
-; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: flat_load_dword v2, v[0:1]
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: .LBB106_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v3, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s0
-; GCN3-NEXT: v_min_i32_e32 v0, s2, v1
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
+; GCN3-NEXT: v_min_i32_e32 v2, s2, v3
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB106_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT: v_mov_b32_e32 v1, s6
-; GCN3-NEXT: v_mov_b32_e32 v2, s7
-; GCN3-NEXT: flat_store_dword v[1:2], v0
+; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v0, s6
+; GCN3-NEXT: v_mov_b32_e32 v1, s7
+; GCN3-NEXT: flat_store_dword v[0:1], v2
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr i32, ptr %out, i32 %index
diff --git a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system.ll b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system.ll
index bcb81cc08b2e65e..bde6d3fe2658f46 100644
--- a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system.ll
@@ -1949,14 +1949,14 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN1-NEXT: v_mov_b32_e32 v4, s35
; GCN1-NEXT: flat_load_dword v2, v[0:1]
; GCN1-NEXT: flat_load_dword v3, v[3:4]
+; GCN1-NEXT: v_mov_b32_e32 v4, s4
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v5, s5
; GCN1-NEXT: .LBB44_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v0, s7, v3
; GCN1-NEXT: v_and_b32_e32 v6, s6, v2
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
; GCN1-NEXT: v_not_b32_e32 v1, v0
; GCN1-NEXT: v_not_b32_e32 v0, v6
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1984,14 +1984,14 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN2-NEXT: v_mov_b32_e32 v4, s35
; GCN2-NEXT: flat_load_dword v2, v[0:1]
; GCN2-NEXT: flat_load_dword v3, v[3:4]
+; GCN2-NEXT: v_mov_b32_e32 v4, s4
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v5, s5
; GCN2-NEXT: .LBB44_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v0, s7, v3
; GCN2-NEXT: v_and_b32_e32 v6, s6, v2
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
; GCN2-NEXT: v_not_b32_e32 v1, v0
; GCN2-NEXT: v_not_b32_e32 v0, v6
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -2014,14 +2014,14 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: .LBB44_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v0, s7, v3
; GCN3-NEXT: v_and_b32_e32 v6, s6, v2
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: v_not_b32_e32 v1, v0
; GCN3-NEXT: v_not_b32_e32 v0, v6
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -2055,14 +2055,12 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: v_mov_b32_e32 v5, s35
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: flat_load_dword v2, v[4:5]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB45_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v0, s7, v3
; GCN1-NEXT: v_and_b32_e32 v6, s6, v2
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
; GCN1-NEXT: v_not_b32_e32 v1, v0
; GCN1-NEXT: v_not_b32_e32 v0, v6
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -2071,12 +2069,12 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB45_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i64_noret_offset_scalar:
@@ -2092,14 +2090,12 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: v_mov_b32_e32 v5, s35
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: flat_load_dword v2, v[4:5]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB45_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v0, s7, v3
; GCN2-NEXT: v_and_b32_e32 v6, s6, v2
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
; GCN2-NEXT: v_not_b32_e32 v1, v0
; GCN2-NEXT: v_not_b32_e32 v0, v6
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -2108,12 +2104,12 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB45_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i64_noret_offset_scalar:
@@ -2122,14 +2118,14 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
+; GCN3-NEXT: v_mov_b32_e32 v4, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: .LBB45_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v0, s7, v3
; GCN3-NEXT: v_and_b32_e32 v6, s6, v2
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: v_not_b32_e32 v1, v0
; GCN3-NEXT: v_not_b32_e32 v0, v6
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -2162,23 +2158,23 @@ define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN1-NEXT: v_mov_b32_e32 v3, s35
; GCN1-NEXT: flat_load_dword v0, v[0:1]
; GCN1-NEXT: flat_load_dword v1, v[2:3]
+; GCN1-NEXT: v_mov_b32_e32 v2, s4
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v3, s5
; GCN1-NEXT: .LBB46_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_and_b32_e32 v0, s7, v3
-; GCN1-NEXT: v_and_b32_e32 v6, s6, v2
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
-; GCN1-NEXT: v_not_b32_e32 v1, v0
-; GCN1-NEXT: v_not_b32_e32 v0, v6
+; GCN1-NEXT: v_mov_b32_e32 v7, v1
+; GCN1-NEXT: v_mov_b32_e32 v6, v0
+; GCN1-NEXT: v_and_b32_e32 v0, s7, v7
+; GCN1-NEXT: v_and_b32_e32 v1, s6, v6
+; GCN1-NEXT: v_not_b32_e32 v5, v0
+; GCN1-NEXT: v_not_b32_e32 v4, v1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB46_1
@@ -2197,23 +2193,23 @@ define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN2-NEXT: v_mov_b32_e32 v3, s35
; GCN2-NEXT: flat_load_dword v0, v[0:1]
; GCN2-NEXT: flat_load_dword v1, v[2:3]
+; GCN2-NEXT: v_mov_b32_e32 v2, s4
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v3, s5
; GCN2-NEXT: .LBB46_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_and_b32_e32 v0, s7, v3
-; GCN2-NEXT: v_and_b32_e32 v6, s6, v2
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
-; GCN2-NEXT: v_not_b32_e32 v1, v0
-; GCN2-NEXT: v_not_b32_e32 v0, v6
+; GCN2-NEXT: v_mov_b32_e32 v7, v1
+; GCN2-NEXT: v_mov_b32_e32 v6, v0
+; GCN2-NEXT: v_and_b32_e32 v0, s7, v7
+; GCN2-NEXT: v_and_b32_e32 v1, s6, v6
+; GCN2-NEXT: v_not_b32_e32 v5, v0
+; GCN2-NEXT: v_not_b32_e32 v4, v1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB46_1
@@ -2227,23 +2223,23 @@ define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: .LBB46_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_and_b32_e32 v0, s7, v3
-; GCN3-NEXT: v_and_b32_e32 v6, s6, v2
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_not_b32_e32 v1, v0
-; GCN3-NEXT: v_not_b32_e32 v0, v6
+; GCN3-NEXT: v_mov_b32_e32 v7, v1
+; GCN3-NEXT: v_mov_b32_e32 v6, v0
+; GCN3-NEXT: v_and_b32_e32 v0, s7, v7
+; GCN3-NEXT: v_and_b32_e32 v1, s6, v6
+; GCN3-NEXT: v_not_b32_e32 v5, v0
+; GCN3-NEXT: v_not_b32_e32 v4, v1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB46_1
@@ -2268,28 +2264,26 @@ define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN1-NEXT: v_mov_b32_e32 v3, s35
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[2:3]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB47_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_and_b32_e32 v0, s7, v3
-; GCN1-NEXT: v_and_b32_e32 v6, s6, v2
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
-; GCN1-NEXT: v_not_b32_e32 v1, v0
-; GCN1-NEXT: v_not_b32_e32 v0, v6
+; GCN1-NEXT: v_mov_b32_e32 v7, v1
+; GCN1-NEXT: v_mov_b32_e32 v6, v0
+; GCN1-NEXT: v_and_b32_e32 v0, s7, v7
+; GCN1-NEXT: v_and_b32_e32 v1, s6, v6
+; GCN1-NEXT: v_not_b32_e32 v5, v0
+; GCN1-NEXT: v_not_b32_e32 v4, v1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB47_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i64_ret_offset_scalar:
@@ -2305,28 +2299,26 @@ define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN2-NEXT: v_mov_b32_e32 v3, s35
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[2:3]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB47_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_and_b32_e32 v0, s7, v3
-; GCN2-NEXT: v_and_b32_e32 v6, s6, v2
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
-; GCN2-NEXT: v_not_b32_e32 v1, v0
-; GCN2-NEXT: v_not_b32_e32 v0, v6
+; GCN2-NEXT: v_mov_b32_e32 v7, v1
+; GCN2-NEXT: v_mov_b32_e32 v6, v0
+; GCN2-NEXT: v_and_b32_e32 v0, s7, v7
+; GCN2-NEXT: v_and_b32_e32 v1, s6, v6
+; GCN2-NEXT: v_not_b32_e32 v5, v0
+; GCN2-NEXT: v_not_b32_e32 v4, v1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB47_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i64_ret_offset_scalar:
@@ -2335,23 +2327,23 @@ define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
+; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: .LBB47_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_and_b32_e32 v0, s7, v3
-; GCN3-NEXT: v_and_b32_e32 v6, s6, v2
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_not_b32_e32 v1, v0
-; GCN3-NEXT: v_not_b32_e32 v0, v6
+; GCN3-NEXT: v_mov_b32_e32 v7, v1
+; GCN3-NEXT: v_mov_b32_e32 v6, v0
+; GCN3-NEXT: v_and_b32_e32 v0, s7, v7
+; GCN3-NEXT: v_and_b32_e32 v1, s6, v6
+; GCN3-NEXT: v_not_b32_e32 v5, v0
+; GCN3-NEXT: v_not_b32_e32 v4, v1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB47_1
@@ -3361,17 +3353,17 @@ define amdgpu_gfx void @flat_atomic_max_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN1-NEXT: v_mov_b32_e32 v4, s35
; GCN1-NEXT: flat_load_dword v2, v[0:1]
; GCN1-NEXT: flat_load_dword v3, v[3:4]
+; GCN1-NEXT: v_mov_b32_e32 v4, s4
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s7
+; GCN1-NEXT: v_mov_b32_e32 v7, s6
+; GCN1-NEXT: v_mov_b32_e32 v5, s5
; GCN1-NEXT: .LBB68_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -3397,17 +3389,17 @@ define amdgpu_gfx void @flat_atomic_max_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN2-NEXT: v_mov_b32_e32 v4, s35
; GCN2-NEXT: flat_load_dword v2, v[0:1]
; GCN2-NEXT: flat_load_dword v3, v[3:4]
+; GCN2-NEXT: v_mov_b32_e32 v4, s4
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s7
+; GCN2-NEXT: v_mov_b32_e32 v7, s6
+; GCN2-NEXT: v_mov_b32_e32 v5, s5
; GCN2-NEXT: .LBB68_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -3428,17 +3420,17 @@ define amdgpu_gfx void @flat_atomic_max_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s7
+; GCN3-NEXT: v_mov_b32_e32 v7, s6
+; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: .LBB68_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -3470,29 +3462,27 @@ define amdgpu_gfx void @flat_atomic_max_i64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: v_mov_b32_e32 v5, s35
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: flat_load_dword v2, v[4:5]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s7
+; GCN1-NEXT: v_mov_b32_e32 v7, s6
; GCN1-NEXT: .LBB69_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB69_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i64_noret_offset_scalar:
@@ -3508,29 +3498,27 @@ define amdgpu_gfx void @flat_atomic_max_i64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: v_mov_b32_e32 v5, s35
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: flat_load_dword v2, v[4:5]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s7
+; GCN2-NEXT: v_mov_b32_e32 v7, s6
; GCN2-NEXT: .LBB69_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB69_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i64_noret_offset_scalar:
@@ -3539,17 +3527,17 @@ define amdgpu_gfx void @flat_atomic_max_i64_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
+; GCN3-NEXT: v_mov_b32_e32 v4, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s7
+; GCN3-NEXT: v_mov_b32_e32 v7, s6
+; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: .LBB69_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -3580,24 +3568,24 @@ define amdgpu_gfx i64 @flat_atomic_max_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN1-NEXT: v_mov_b32_e32 v3, s35
; GCN1-NEXT: flat_load_dword v0, v[0:1]
; GCN1-NEXT: flat_load_dword v1, v[2:3]
+; GCN1-NEXT: v_mov_b32_e32 v2, s4
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s7
+; GCN1-NEXT: v_mov_b32_e32 v5, s6
+; GCN1-NEXT: v_mov_b32_e32 v3, s5
; GCN1-NEXT: .LBB70_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_mov_b32_e32 v9, v1
+; GCN1-NEXT: v_mov_b32_e32 v8, v0
+; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB70_1
@@ -3616,24 +3604,24 @@ define amdgpu_gfx i64 @flat_atomic_max_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN2-NEXT: v_mov_b32_e32 v3, s35
; GCN2-NEXT: flat_load_dword v0, v[0:1]
; GCN2-NEXT: flat_load_dword v1, v[2:3]
+; GCN2-NEXT: v_mov_b32_e32 v2, s4
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s7
+; GCN2-NEXT: v_mov_b32_e32 v5, s6
+; GCN2-NEXT: v_mov_b32_e32 v3, s5
; GCN2-NEXT: .LBB70_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_mov_b32_e32 v9, v1
+; GCN2-NEXT: v_mov_b32_e32 v8, v0
+; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB70_1
@@ -3647,24 +3635,24 @@ define amdgpu_gfx i64 @flat_atomic_max_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s7
+; GCN3-NEXT: v_mov_b32_e32 v5, s6
+; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: .LBB70_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_mov_b32_e32 v9, v1
+; GCN3-NEXT: v_mov_b32_e32 v8, v0
+; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB70_1
@@ -3689,29 +3677,27 @@ define amdgpu_gfx i64 @flat_atomic_max_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN1-NEXT: v_mov_b32_e32 v3, s35
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[2:3]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s7
+; GCN1-NEXT: v_mov_b32_e32 v5, s6
; GCN1-NEXT: .LBB71_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_mov_b32_e32 v9, v1
+; GCN1-NEXT: v_mov_b32_e32 v8, v0
+; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB71_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i64_ret_offset_scalar:
@@ -3727,29 +3713,27 @@ define amdgpu_gfx i64 @flat_atomic_max_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN2-NEXT: v_mov_b32_e32 v3, s35
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[2:3]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s7
+; GCN2-NEXT: v_mov_b32_e32 v5, s6
; GCN2-NEXT: .LBB71_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_mov_b32_e32 v9, v1
+; GCN2-NEXT: v_mov_b32_e32 v8, v0
+; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB71_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i64_ret_offset_scalar:
@@ -3758,24 +3742,24 @@ define amdgpu_gfx i64 @flat_atomic_max_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
+; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s7
+; GCN3-NEXT: v_mov_b32_e32 v5, s6
+; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: .LBB71_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_mov_b32_e32 v9, v1
+; GCN3-NEXT: v_mov_b32_e32 v8, v0
+; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB71_1
@@ -3798,29 +3782,27 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_add_u32 s0, s0, 32
; GCN1-NEXT: s_addc_u32 s1, s1, 0
-; GCN1-NEXT: v_mov_b32_e32 v0, s0
-; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; GCN1-NEXT: s_mov_b64 s[4:5], 0
+; GCN1-NEXT: v_mov_b32_e32 v5, s1
+; GCN1-NEXT: v_mov_b32_e32 v4, s0
+; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s3
+; GCN1-NEXT: v_mov_b32_e32 v7, s2
; GCN1-NEXT: .LBB72_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s3
-; GCN1-NEXT: v_mov_b32_e32 v6, s2
-; GCN1-NEXT: v_mov_b32_e32 v5, s1
-; GCN1-NEXT: v_mov_b32_e32 v4, s0
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB72_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_endpgm
@@ -3835,29 +3817,27 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_add_u32 s0, s0, 32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
-; GCN2-NEXT: v_mov_b32_e32 v0, s0
-; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; GCN2-NEXT: s_mov_b64 s[4:5], 0
+; GCN2-NEXT: v_mov_b32_e32 v5, s1
+; GCN2-NEXT: v_mov_b32_e32 v4, s0
+; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s3
+; GCN2-NEXT: v_mov_b32_e32 v7, s2
; GCN2-NEXT: .LBB72_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s3
-; GCN2-NEXT: v_mov_b32_e32 v6, s2
-; GCN2-NEXT: v_mov_b32_e32 v5, s1
-; GCN2-NEXT: v_mov_b32_e32 v4, s0
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB72_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_endpgm
@@ -3870,29 +3850,27 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN3-NEXT: s_lshl_b64 s[0:1], s[2:3], 3
; GCN3-NEXT: s_add_u32 s0, s4, s0
; GCN3-NEXT: s_addc_u32 s1, s5, s1
-; GCN3-NEXT: v_mov_b32_e32 v0, s0
-; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
-; GCN3-NEXT: s_mov_b64 s[2:3], 0
+; GCN3-NEXT: v_mov_b32_e32 v5, s1
+; GCN3-NEXT: v_mov_b32_e32 v4, s0
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] offset:32
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s7
+; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: .LBB72_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v5, s1
-; GCN3-NEXT: v_mov_b32_e32 v4, s0
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB72_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_endpgm
@@ -3915,33 +3893,31 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GCN1-NEXT: s_mov_b64 s[6:7], 0
+; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s5
+; GCN1-NEXT: v_mov_b32_e32 v5, s4
; GCN1-NEXT: .LBB73_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s5
-; GCN1-NEXT: v_mov_b32_e32 v6, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s1
-; GCN1-NEXT: v_mov_b32_e32 v4, s0
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_mov_b32_e32 v9, v3
+; GCN1-NEXT: v_mov_b32_e32 v8, v2
+; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB73_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN1-NEXT: v_mov_b32_e32 v2, s2
-; GCN1-NEXT: v_mov_b32_e32 v3, s3
-; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v0, s2
+; GCN1-NEXT: v_mov_b32_e32 v1, s3
+; GCN1-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_max_i64_ret_addr64_offset:
@@ -3955,33 +3931,31 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GCN2-NEXT: s_mov_b64 s[6:7], 0
+; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s5
+; GCN2-NEXT: v_mov_b32_e32 v5, s4
; GCN2-NEXT: .LBB73_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s5
-; GCN2-NEXT: v_mov_b32_e32 v6, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s1
-; GCN2-NEXT: v_mov_b32_e32 v4, s0
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_mov_b32_e32 v9, v3
+; GCN2-NEXT: v_mov_b32_e32 v8, v2
+; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB73_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN2-NEXT: v_mov_b32_e32 v2, s2
-; GCN2-NEXT: v_mov_b32_e32 v3, s3
-; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v0, s2
+; GCN2-NEXT: v_mov_b32_e32 v1, s3
+; GCN2-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_max_i64_ret_addr64_offset:
@@ -3993,33 +3967,31 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN3-NEXT: s_addc_u32 s1, s1, s7
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
-; GCN3-NEXT: s_mov_b64 s[6:7], 0
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s5
+; GCN3-NEXT: v_mov_b32_e32 v5, s4
; GCN3-NEXT: .LBB73_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s5
-; GCN3-NEXT: v_mov_b32_e32 v6, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s1
-; GCN3-NEXT: v_mov_b32_e32 v4, s0
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_mov_b32_e32 v9, v3
+; GCN3-NEXT: v_mov_b32_e32 v8, v2
+; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB73_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN3-NEXT: v_mov_b32_e32 v2, s2
-; GCN3-NEXT: v_mov_b32_e32 v3, s3
-; GCN3-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v0, s2
+; GCN3-NEXT: v_mov_b32_e32 v1, s3
+; GCN3-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr i64, ptr %out, i64 %index
@@ -4038,29 +4010,27 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN1-NEXT: s_lshl_b64 s[4:5], s[4:5], 3
; GCN1-NEXT: s_add_u32 s0, s0, s4
; GCN1-NEXT: s_addc_u32 s1, s1, s5
-; GCN1-NEXT: v_mov_b32_e32 v0, s0
-; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; GCN1-NEXT: s_mov_b64 s[4:5], 0
+; GCN1-NEXT: v_mov_b32_e32 v5, s1
+; GCN1-NEXT: v_mov_b32_e32 v4, s0
+; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s3
+; GCN1-NEXT: v_mov_b32_e32 v7, s2
; GCN1-NEXT: .LBB74_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s3
-; GCN1-NEXT: v_mov_b32_e32 v6, s2
-; GCN1-NEXT: v_mov_b32_e32 v5, s1
-; GCN1-NEXT: v_mov_b32_e32 v4, s0
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB74_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_endpgm
@@ -4073,29 +4043,27 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN2-NEXT: s_lshl_b64 s[4:5], s[4:5], 3
; GCN2-NEXT: s_add_u32 s0, s0, s4
; GCN2-NEXT: s_addc_u32 s1, s1, s5
-; GCN2-NEXT: v_mov_b32_e32 v0, s0
-; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; GCN2-NEXT: s_mov_b64 s[4:5], 0
+; GCN2-NEXT: v_mov_b32_e32 v5, s1
+; GCN2-NEXT: v_mov_b32_e32 v4, s0
+; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s3
+; GCN2-NEXT: v_mov_b32_e32 v7, s2
; GCN2-NEXT: .LBB74_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s3
-; GCN2-NEXT: v_mov_b32_e32 v6, s2
-; GCN2-NEXT: v_mov_b32_e32 v5, s1
-; GCN2-NEXT: v_mov_b32_e32 v4, s0
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB74_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_endpgm
@@ -4108,29 +4076,27 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN3-NEXT: s_lshl_b64 s[0:1], s[2:3], 3
; GCN3-NEXT: s_add_u32 s0, s4, s0
; GCN3-NEXT: s_addc_u32 s1, s5, s1
-; GCN3-NEXT: v_mov_b32_e32 v0, s0
-; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; GCN3-NEXT: s_mov_b64 s[2:3], 0
+; GCN3-NEXT: v_mov_b32_e32 v5, s1
+; GCN3-NEXT: v_mov_b32_e32 v4, s0
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s7
+; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: .LBB74_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v5, s1
-; GCN3-NEXT: v_mov_b32_e32 v4, s0
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB74_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_endpgm
@@ -4150,33 +4116,31 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN1-NEXT: s_addc_u32 s1, s1, s7
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GCN1-NEXT: s_mov_b64 s[6:7], 0
+; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s5
+; GCN1-NEXT: v_mov_b32_e32 v5, s4
; GCN1-NEXT: .LBB75_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s5
-; GCN1-NEXT: v_mov_b32_e32 v6, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s1
-; GCN1-NEXT: v_mov_b32_e32 v4, s0
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_mov_b32_e32 v9, v3
+; GCN1-NEXT: v_mov_b32_e32 v8, v2
+; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB75_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN1-NEXT: v_mov_b32_e32 v2, s2
-; GCN1-NEXT: v_mov_b32_e32 v3, s3
-; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v0, s2
+; GCN1-NEXT: v_mov_b32_e32 v1, s3
+; GCN1-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_max_i64_ret_addr64:
@@ -4188,33 +4152,31 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GCN2-NEXT: s_mov_b64 s[6:7], 0
+; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s5
+; GCN2-NEXT: v_mov_b32_e32 v5, s4
; GCN2-NEXT: .LBB75_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s5
-; GCN2-NEXT: v_mov_b32_e32 v6, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s1
-; GCN2-NEXT: v_mov_b32_e32 v4, s0
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_mov_b32_e32 v9, v3
+; GCN2-NEXT: v_mov_b32_e32 v8, v2
+; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB75_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN2-NEXT: v_mov_b32_e32 v2, s2
-; GCN2-NEXT: v_mov_b32_e32 v3, s3
-; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v0, s2
+; GCN2-NEXT: v_mov_b32_e32 v1, s3
+; GCN2-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_max_i64_ret_addr64:
@@ -4226,33 +4188,31 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN3-NEXT: s_addc_u32 s1, s1, s7
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GCN3-NEXT: s_mov_b64 s[6:7], 0
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s5
+; GCN3-NEXT: v_mov_b32_e32 v5, s4
; GCN3-NEXT: .LBB75_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s5
-; GCN3-NEXT: v_mov_b32_e32 v6, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s1
-; GCN3-NEXT: v_mov_b32_e32 v4, s0
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_mov_b32_e32 v9, v3
+; GCN3-NEXT: v_mov_b32_e32 v8, v2
+; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB75_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN3-NEXT: v_mov_b32_e32 v2, s2
-; GCN3-NEXT: v_mov_b32_e32 v3, s3
-; GCN3-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v0, s2
+; GCN3-NEXT: v_mov_b32_e32 v1, s3
+; GCN3-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr i64, ptr %out, i64 %index
@@ -4635,17 +4595,17 @@ define amdgpu_gfx void @flat_atomic_umax_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN1-NEXT: v_mov_b32_e32 v4, s35
; GCN1-NEXT: flat_load_dword v2, v[0:1]
; GCN1-NEXT: flat_load_dword v3, v[3:4]
+; GCN1-NEXT: v_mov_b32_e32 v4, s4
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s7
+; GCN1-NEXT: v_mov_b32_e32 v7, s6
+; GCN1-NEXT: v_mov_b32_e32 v5, s5
; GCN1-NEXT: .LBB80_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -4671,17 +4631,17 @@ define amdgpu_gfx void @flat_atomic_umax_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN2-NEXT: v_mov_b32_e32 v4, s35
; GCN2-NEXT: flat_load_dword v2, v[0:1]
; GCN2-NEXT: flat_load_dword v3, v[3:4]
+; GCN2-NEXT: v_mov_b32_e32 v4, s4
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s7
+; GCN2-NEXT: v_mov_b32_e32 v7, s6
+; GCN2-NEXT: v_mov_b32_e32 v5, s5
; GCN2-NEXT: .LBB80_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -4702,17 +4662,17 @@ define amdgpu_gfx void @flat_atomic_umax_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s7
+; GCN3-NEXT: v_mov_b32_e32 v7, s6
+; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: .LBB80_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -4744,29 +4704,27 @@ define amdgpu_gfx void @flat_atomic_umax_i64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: v_mov_b32_e32 v5, s35
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: flat_load_dword v2, v[4:5]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s7
+; GCN1-NEXT: v_mov_b32_e32 v7, s6
; GCN1-NEXT: .LBB81_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB81_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i64_noret_offset_scalar:
@@ -4782,29 +4740,27 @@ define amdgpu_gfx void @flat_atomic_umax_i64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: v_mov_b32_e32 v5, s35
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: flat_load_dword v2, v[4:5]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s7
+; GCN2-NEXT: v_mov_b32_e32 v7, s6
; GCN2-NEXT: .LBB81_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB81_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i64_noret_offset_scalar:
@@ -4813,17 +4769,17 @@ define amdgpu_gfx void @flat_atomic_umax_i64_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
+; GCN3-NEXT: v_mov_b32_e32 v4, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s7
+; GCN3-NEXT: v_mov_b32_e32 v7, s6
+; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: .LBB81_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -4854,24 +4810,24 @@ define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN1-NEXT: v_mov_b32_e32 v3, s35
; GCN1-NEXT: flat_load_dword v0, v[0:1]
; GCN1-NEXT: flat_load_dword v1, v[2:3]
+; GCN1-NEXT: v_mov_b32_e32 v2, s4
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s7
+; GCN1-NEXT: v_mov_b32_e32 v5, s6
+; GCN1-NEXT: v_mov_b32_e32 v3, s5
; GCN1-NEXT: .LBB82_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_mov_b32_e32 v9, v1
+; GCN1-NEXT: v_mov_b32_e32 v8, v0
+; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB82_1
@@ -4890,24 +4846,24 @@ define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN2-NEXT: v_mov_b32_e32 v3, s35
; GCN2-NEXT: flat_load_dword v0, v[0:1]
; GCN2-NEXT: flat_load_dword v1, v[2:3]
+; GCN2-NEXT: v_mov_b32_e32 v2, s4
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s7
+; GCN2-NEXT: v_mov_b32_e32 v5, s6
+; GCN2-NEXT: v_mov_b32_e32 v3, s5
; GCN2-NEXT: .LBB82_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_mov_b32_e32 v9, v1
+; GCN2-NEXT: v_mov_b32_e32 v8, v0
+; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB82_1
@@ -4921,24 +4877,24 @@ define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s7
+; GCN3-NEXT: v_mov_b32_e32 v5, s6
+; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: .LBB82_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_mov_b32_e32 v9, v1
+; GCN3-NEXT: v_mov_b32_e32 v8, v0
+; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB82_1
@@ -4963,29 +4919,27 @@ define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN1-NEXT: v_mov_b32_e32 v3, s35
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[2:3]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s7
+; GCN1-NEXT: v_mov_b32_e32 v5, s6
; GCN1-NEXT: .LBB83_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_mov_b32_e32 v9, v1
+; GCN1-NEXT: v_mov_b32_e32 v8, v0
+; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB83_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i64_ret_offset_scalar:
@@ -5001,29 +4955,27 @@ define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN2-NEXT: v_mov_b32_e32 v3, s35
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[2:3]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s7
+; GCN2-NEXT: v_mov_b32_e32 v5, s6
; GCN2-NEXT: .LBB83_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_mov_b32_e32 v9, v1
+; GCN2-NEXT: v_mov_b32_e32 v8, v0
+; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB83_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i64_ret_offset_scalar:
@@ -5032,24 +4984,24 @@ define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
+; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s7
+; GCN3-NEXT: v_mov_b32_e32 v5, s6
+; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: .LBB83_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_mov_b32_e32 v9, v1
+; GCN3-NEXT: v_mov_b32_e32 v8, v0
+; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB83_1
@@ -5072,29 +5024,27 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_add_u32 s0, s0, 32
; GCN1-NEXT: s_addc_u32 s1, s1, 0
-; GCN1-NEXT: v_mov_b32_e32 v0, s0
-; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; GCN1-NEXT: s_mov_b64 s[4:5], 0
+; GCN1-NEXT: v_mov_b32_e32 v5, s1
+; GCN1-NEXT: v_mov_b32_e32 v4, s0
+; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s3
+; GCN1-NEXT: v_mov_b32_e32 v7, s2
; GCN1-NEXT: .LBB84_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s3
-; GCN1-NEXT: v_mov_b32_e32 v6, s2
-; GCN1-NEXT: v_mov_b32_e32 v5, s1
-; GCN1-NEXT: v_mov_b32_e32 v4, s0
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB84_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_endpgm
@@ -5109,29 +5059,27 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_add_u32 s0, s0, 32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
-; GCN2-NEXT: v_mov_b32_e32 v0, s0
-; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; GCN2-NEXT: s_mov_b64 s[4:5], 0
+; GCN2-NEXT: v_mov_b32_e32 v5, s1
+; GCN2-NEXT: v_mov_b32_e32 v4, s0
+; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s3
+; GCN2-NEXT: v_mov_b32_e32 v7, s2
; GCN2-NEXT: .LBB84_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s3
-; GCN2-NEXT: v_mov_b32_e32 v6, s2
-; GCN2-NEXT: v_mov_b32_e32 v5, s1
-; GCN2-NEXT: v_mov_b32_e32 v4, s0
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB84_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_endpgm
@@ -5144,29 +5092,27 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64
; GCN3-NEXT: s_lshl_b64 s[0:1], s[2:3], 3
; GCN3-NEXT: s_add_u32 s0, s4, s0
; GCN3-NEXT: s_addc_u32 s1, s5, s1
-; GCN3-NEXT: v_mov_b32_e32 v0, s0
-; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
-; GCN3-NEXT: s_mov_b64 s[2:3], 0
+; GCN3-NEXT: v_mov_b32_e32 v5, s1
+; GCN3-NEXT: v_mov_b32_e32 v4, s0
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] offset:32
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s7
+; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: .LBB84_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v5, s1
-; GCN3-NEXT: v_mov_b32_e32 v4, s0
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB84_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_endpgm
@@ -5189,33 +5135,31 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GCN1-NEXT: s_mov_b64 s[6:7], 0
+; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s5
+; GCN1-NEXT: v_mov_b32_e32 v5, s4
; GCN1-NEXT: .LBB85_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s5
-; GCN1-NEXT: v_mov_b32_e32 v6, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s1
-; GCN1-NEXT: v_mov_b32_e32 v4, s0
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_mov_b32_e32 v9, v3
+; GCN1-NEXT: v_mov_b32_e32 v8, v2
+; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB85_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN1-NEXT: v_mov_b32_e32 v2, s2
-; GCN1-NEXT: v_mov_b32_e32 v3, s3
-; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v0, s2
+; GCN1-NEXT: v_mov_b32_e32 v1, s3
+; GCN1-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_umax_i64_ret_addr64_offset:
@@ -5229,33 +5173,31 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GCN2-NEXT: s_mov_b64 s[6:7], 0
+; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s5
+; GCN2-NEXT: v_mov_b32_e32 v5, s4
; GCN2-NEXT: .LBB85_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s5
-; GCN2-NEXT: v_mov_b32_e32 v6, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s1
-; GCN2-NEXT: v_mov_b32_e32 v4, s0
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_mov_b32_e32 v9, v3
+; GCN2-NEXT: v_mov_b32_e32 v8, v2
+; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB85_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN2-NEXT: v_mov_b32_e32 v2, s2
-; GCN2-NEXT: v_mov_b32_e32 v3, s3
-; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v0, s2
+; GCN2-NEXT: v_mov_b32_e32 v1, s3
+; GCN2-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_umax_i64_ret_addr64_offset:
@@ -5267,33 +5209,31 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2
; GCN3-NEXT: s_addc_u32 s1, s1, s7
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
-; GCN3-NEXT: s_mov_b64 s[6:7], 0
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s5
+; GCN3-NEXT: v_mov_b32_e32 v5, s4
; GCN3-NEXT: .LBB85_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s5
-; GCN3-NEXT: v_mov_b32_e32 v6, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s1
-; GCN3-NEXT: v_mov_b32_e32 v4, s0
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_mov_b32_e32 v9, v3
+; GCN3-NEXT: v_mov_b32_e32 v8, v2
+; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB85_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN3-NEXT: v_mov_b32_e32 v2, s2
-; GCN3-NEXT: v_mov_b32_e32 v3, s3
-; GCN3-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v0, s2
+; GCN3-NEXT: v_mov_b32_e32 v1, s3
+; GCN3-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr i64, ptr %out, i64 %index
@@ -5313,33 +5253,31 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %
; GCN1-NEXT: s_addc_u32 s1, s1, s7
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GCN1-NEXT: s_mov_b64 s[6:7], 0
+; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s5
+; GCN1-NEXT: v_mov_b32_e32 v5, s4
; GCN1-NEXT: .LBB86_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s5
-; GCN1-NEXT: v_mov_b32_e32 v6, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s1
-; GCN1-NEXT: v_mov_b32_e32 v4, s0
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_mov_b32_e32 v9, v3
+; GCN1-NEXT: v_mov_b32_e32 v8, v2
+; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB86_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN1-NEXT: v_mov_b32_e32 v2, s2
-; GCN1-NEXT: v_mov_b32_e32 v3, s3
-; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v0, s2
+; GCN1-NEXT: v_mov_b32_e32 v1, s3
+; GCN1-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_umax_i64_ret_addr64:
@@ -5351,33 +5289,31 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %
; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GCN2-NEXT: s_mov_b64 s[6:7], 0
+; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s5
+; GCN2-NEXT: v_mov_b32_e32 v5, s4
; GCN2-NEXT: .LBB86_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s5
-; GCN2-NEXT: v_mov_b32_e32 v6, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s1
-; GCN2-NEXT: v_mov_b32_e32 v4, s0
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_mov_b32_e32 v9, v3
+; GCN2-NEXT: v_mov_b32_e32 v8, v2
+; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB86_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN2-NEXT: v_mov_b32_e32 v2, s2
-; GCN2-NEXT: v_mov_b32_e32 v3, s3
-; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v0, s2
+; GCN2-NEXT: v_mov_b32_e32 v1, s3
+; GCN2-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_umax_i64_ret_addr64:
@@ -5389,33 +5325,31 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %
; GCN3-NEXT: s_addc_u32 s1, s1, s7
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GCN3-NEXT: s_mov_b64 s[6:7], 0
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s5
+; GCN3-NEXT: v_mov_b32_e32 v5, s4
; GCN3-NEXT: .LBB86_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s5
-; GCN3-NEXT: v_mov_b32_e32 v6, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s1
-; GCN3-NEXT: v_mov_b32_e32 v4, s0
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_mov_b32_e32 v9, v3
+; GCN3-NEXT: v_mov_b32_e32 v8, v2
+; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB86_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN3-NEXT: v_mov_b32_e32 v2, s2
-; GCN3-NEXT: v_mov_b32_e32 v3, s3
-; GCN3-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v0, s2
+; GCN3-NEXT: v_mov_b32_e32 v1, s3
+; GCN3-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr i64, ptr %out, i64 %index
@@ -5798,17 +5732,17 @@ define amdgpu_gfx void @flat_atomic_umin_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN1-NEXT: v_mov_b32_e32 v4, s35
; GCN1-NEXT: flat_load_dword v2, v[0:1]
; GCN1-NEXT: flat_load_dword v3, v[3:4]
+; GCN1-NEXT: v_mov_b32_e32 v4, s4
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s7
+; GCN1-NEXT: v_mov_b32_e32 v7, s6
+; GCN1-NEXT: v_mov_b32_e32 v5, s5
; GCN1-NEXT: .LBB91_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -5834,17 +5768,17 @@ define amdgpu_gfx void @flat_atomic_umin_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN2-NEXT: v_mov_b32_e32 v4, s35
; GCN2-NEXT: flat_load_dword v2, v[0:1]
; GCN2-NEXT: flat_load_dword v3, v[3:4]
+; GCN2-NEXT: v_mov_b32_e32 v4, s4
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s7
+; GCN2-NEXT: v_mov_b32_e32 v7, s6
+; GCN2-NEXT: v_mov_b32_e32 v5, s5
; GCN2-NEXT: .LBB91_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -5865,17 +5799,17 @@ define amdgpu_gfx void @flat_atomic_umin_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s7
+; GCN3-NEXT: v_mov_b32_e32 v7, s6
+; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: .LBB91_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -5907,29 +5841,27 @@ define amdgpu_gfx void @flat_atomic_umin_i64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: v_mov_b32_e32 v5, s35
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: flat_load_dword v2, v[4:5]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s7
+; GCN1-NEXT: v_mov_b32_e32 v7, s6
; GCN1-NEXT: .LBB92_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB92_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i64_noret_offset_scalar:
@@ -5945,29 +5877,27 @@ define amdgpu_gfx void @flat_atomic_umin_i64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: v_mov_b32_e32 v5, s35
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: flat_load_dword v2, v[4:5]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s7
+; GCN2-NEXT: v_mov_b32_e32 v7, s6
; GCN2-NEXT: .LBB92_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB92_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i64_noret_offset_scalar:
@@ -5976,17 +5906,17 @@ define amdgpu_gfx void @flat_atomic_umin_i64_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
+; GCN3-NEXT: v_mov_b32_e32 v4, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s7
+; GCN3-NEXT: v_mov_b32_e32 v7, s6
+; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: .LBB92_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -6017,24 +5947,24 @@ define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN1-NEXT: v_mov_b32_e32 v3, s35
; GCN1-NEXT: flat_load_dword v0, v[0:1]
; GCN1-NEXT: flat_load_dword v1, v[2:3]
+; GCN1-NEXT: v_mov_b32_e32 v2, s4
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s7
+; GCN1-NEXT: v_mov_b32_e32 v5, s6
+; GCN1-NEXT: v_mov_b32_e32 v3, s5
; GCN1-NEXT: .LBB93_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_mov_b32_e32 v9, v1
+; GCN1-NEXT: v_mov_b32_e32 v8, v0
+; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB93_1
@@ -6053,24 +5983,24 @@ define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN2-NEXT: v_mov_b32_e32 v3, s35
; GCN2-NEXT: flat_load_dword v0, v[0:1]
; GCN2-NEXT: flat_load_dword v1, v[2:3]
+; GCN2-NEXT: v_mov_b32_e32 v2, s4
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s7
+; GCN2-NEXT: v_mov_b32_e32 v5, s6
+; GCN2-NEXT: v_mov_b32_e32 v3, s5
; GCN2-NEXT: .LBB93_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_mov_b32_e32 v9, v1
+; GCN2-NEXT: v_mov_b32_e32 v8, v0
+; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB93_1
@@ -6084,24 +6014,24 @@ define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s7
+; GCN3-NEXT: v_mov_b32_e32 v5, s6
+; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: .LBB93_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_mov_b32_e32 v9, v1
+; GCN3-NEXT: v_mov_b32_e32 v8, v0
+; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB93_1
@@ -6126,29 +6056,27 @@ define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN1-NEXT: v_mov_b32_e32 v3, s35
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[2:3]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s7
+; GCN1-NEXT: v_mov_b32_e32 v5, s6
; GCN1-NEXT: .LBB94_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_mov_b32_e32 v9, v1
+; GCN1-NEXT: v_mov_b32_e32 v8, v0
+; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB94_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i64_ret_offset_scalar:
@@ -6164,29 +6092,27 @@ define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN2-NEXT: v_mov_b32_e32 v3, s35
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[2:3]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s7
+; GCN2-NEXT: v_mov_b32_e32 v5, s6
; GCN2-NEXT: .LBB94_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_mov_b32_e32 v9, v1
+; GCN2-NEXT: v_mov_b32_e32 v8, v0
+; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB94_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i64_ret_offset_scalar:
@@ -6195,24 +6121,24 @@ define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
+; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s7
+; GCN3-NEXT: v_mov_b32_e32 v5, s6
+; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: .LBB94_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_mov_b32_e32 v9, v1
+; GCN3-NEXT: v_mov_b32_e32 v8, v0
+; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB94_1
@@ -6598,17 +6524,17 @@ define amdgpu_gfx void @flat_atomic_min_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN1-NEXT: v_mov_b32_e32 v4, s35
; GCN1-NEXT: flat_load_dword v2, v[0:1]
; GCN1-NEXT: flat_load_dword v3, v[3:4]
+; GCN1-NEXT: v_mov_b32_e32 v4, s4
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s7
+; GCN1-NEXT: v_mov_b32_e32 v7, s6
+; GCN1-NEXT: v_mov_b32_e32 v5, s5
; GCN1-NEXT: .LBB99_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -6634,17 +6560,17 @@ define amdgpu_gfx void @flat_atomic_min_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN2-NEXT: v_mov_b32_e32 v4, s35
; GCN2-NEXT: flat_load_dword v2, v[0:1]
; GCN2-NEXT: flat_load_dword v3, v[3:4]
+; GCN2-NEXT: v_mov_b32_e32 v4, s4
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s7
+; GCN2-NEXT: v_mov_b32_e32 v7, s6
+; GCN2-NEXT: v_mov_b32_e32 v5, s5
; GCN2-NEXT: .LBB99_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -6665,17 +6591,17 @@ define amdgpu_gfx void @flat_atomic_min_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s7
+; GCN3-NEXT: v_mov_b32_e32 v7, s6
+; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: .LBB99_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -6707,29 +6633,27 @@ define amdgpu_gfx void @flat_atomic_min_i64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: v_mov_b32_e32 v5, s35
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: flat_load_dword v2, v[4:5]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s7
+; GCN1-NEXT: v_mov_b32_e32 v7, s6
; GCN1-NEXT: .LBB100_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB100_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i64_noret_offset_scalar:
@@ -6745,29 +6669,27 @@ define amdgpu_gfx void @flat_atomic_min_i64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: v_mov_b32_e32 v5, s35
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: flat_load_dword v2, v[4:5]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s7
+; GCN2-NEXT: v_mov_b32_e32 v7, s6
; GCN2-NEXT: .LBB100_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB100_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i64_noret_offset_scalar:
@@ -6776,17 +6698,17 @@ define amdgpu_gfx void @flat_atomic_min_i64_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
+; GCN3-NEXT: v_mov_b32_e32 v4, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s7
+; GCN3-NEXT: v_mov_b32_e32 v7, s6
+; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: .LBB100_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -6817,24 +6739,24 @@ define amdgpu_gfx i64 @flat_atomic_min_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN1-NEXT: v_mov_b32_e32 v3, s35
; GCN1-NEXT: flat_load_dword v0, v[0:1]
; GCN1-NEXT: flat_load_dword v1, v[2:3]
+; GCN1-NEXT: v_mov_b32_e32 v2, s4
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s7
+; GCN1-NEXT: v_mov_b32_e32 v5, s6
+; GCN1-NEXT: v_mov_b32_e32 v3, s5
; GCN1-NEXT: .LBB101_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_mov_b32_e32 v9, v1
+; GCN1-NEXT: v_mov_b32_e32 v8, v0
+; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB101_1
@@ -6853,24 +6775,24 @@ define amdgpu_gfx i64 @flat_atomic_min_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN2-NEXT: v_mov_b32_e32 v3, s35
; GCN2-NEXT: flat_load_dword v0, v[0:1]
; GCN2-NEXT: flat_load_dword v1, v[2:3]
+; GCN2-NEXT: v_mov_b32_e32 v2, s4
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s7
+; GCN2-NEXT: v_mov_b32_e32 v5, s6
+; GCN2-NEXT: v_mov_b32_e32 v3, s5
; GCN2-NEXT: .LBB101_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_mov_b32_e32 v9, v1
+; GCN2-NEXT: v_mov_b32_e32 v8, v0
+; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB101_1
@@ -6884,24 +6806,24 @@ define amdgpu_gfx i64 @flat_atomic_min_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s7
+; GCN3-NEXT: v_mov_b32_e32 v5, s6
+; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: .LBB101_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_mov_b32_e32 v9, v1
+; GCN3-NEXT: v_mov_b32_e32 v8, v0
+; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB101_1
@@ -6926,29 +6848,27 @@ define amdgpu_gfx i64 @flat_atomic_min_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN1-NEXT: v_mov_b32_e32 v3, s35
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[2:3]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s7
+; GCN1-NEXT: v_mov_b32_e32 v5, s6
; GCN1-NEXT: .LBB102_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_mov_b32_e32 v9, v1
+; GCN1-NEXT: v_mov_b32_e32 v8, v0
+; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB102_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i64_ret_offset_scalar:
@@ -6964,29 +6884,27 @@ define amdgpu_gfx i64 @flat_atomic_min_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN2-NEXT: v_mov_b32_e32 v3, s35
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[2:3]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s7
+; GCN2-NEXT: v_mov_b32_e32 v5, s6
; GCN2-NEXT: .LBB102_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_mov_b32_e32 v9, v1
+; GCN2-NEXT: v_mov_b32_e32 v8, v0
+; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB102_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i64_ret_offset_scalar:
@@ -6995,24 +6913,24 @@ define amdgpu_gfx i64 @flat_atomic_min_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
+; GCN3-NEXT: v_mov_b32_e32 v2, s4
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s7
+; GCN3-NEXT: v_mov_b32_e32 v5, s6
+; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: .LBB102_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_mov_b32_e32 v9, v1
+; GCN3-NEXT: v_mov_b32_e32 v8, v0
+; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB102_1
@@ -7035,29 +6953,27 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_add_u32 s0, s0, 32
; GCN1-NEXT: s_addc_u32 s1, s1, 0
-; GCN1-NEXT: v_mov_b32_e32 v0, s0
-; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; GCN1-NEXT: s_mov_b64 s[4:5], 0
+; GCN1-NEXT: v_mov_b32_e32 v5, s1
+; GCN1-NEXT: v_mov_b32_e32 v4, s0
+; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s3
+; GCN1-NEXT: v_mov_b32_e32 v7, s2
; GCN1-NEXT: .LBB103_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s3
-; GCN1-NEXT: v_mov_b32_e32 v6, s2
-; GCN1-NEXT: v_mov_b32_e32 v5, s1
-; GCN1-NEXT: v_mov_b32_e32 v4, s0
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB103_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_endpgm
@@ -7072,29 +6988,27 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_add_u32 s0, s0, 32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
-; GCN2-NEXT: v_mov_b32_e32 v0, s0
-; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; GCN2-NEXT: s_mov_b64 s[4:5], 0
+; GCN2-NEXT: v_mov_b32_e32 v5, s1
+; GCN2-NEXT: v_mov_b32_e32 v4, s0
+; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s3
+; GCN2-NEXT: v_mov_b32_e32 v7, s2
; GCN2-NEXT: .LBB103_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s3
-; GCN2-NEXT: v_mov_b32_e32 v6, s2
-; GCN2-NEXT: v_mov_b32_e32 v5, s1
-; GCN2-NEXT: v_mov_b32_e32 v4, s0
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB103_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_endpgm
@@ -7107,29 +7021,27 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN3-NEXT: s_lshl_b64 s[0:1], s[2:3], 3
; GCN3-NEXT: s_add_u32 s0, s4, s0
; GCN3-NEXT: s_addc_u32 s1, s5, s1
-; GCN3-NEXT: v_mov_b32_e32 v0, s0
-; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
-; GCN3-NEXT: s_mov_b64 s[2:3], 0
+; GCN3-NEXT: v_mov_b32_e32 v5, s1
+; GCN3-NEXT: v_mov_b32_e32 v4, s0
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5] offset:32
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s7
+; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: .LBB103_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v5, s1
-; GCN3-NEXT: v_mov_b32_e32 v4, s0
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB103_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_endpgm
@@ -7152,33 +7064,31 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GCN1-NEXT: s_mov_b64 s[6:7], 0
+; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s5
+; GCN1-NEXT: v_mov_b32_e32 v5, s4
; GCN1-NEXT: .LBB104_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s5
-; GCN1-NEXT: v_mov_b32_e32 v6, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s1
-; GCN1-NEXT: v_mov_b32_e32 v4, s0
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_mov_b32_e32 v9, v3
+; GCN1-NEXT: v_mov_b32_e32 v8, v2
+; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB104_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN1-NEXT: v_mov_b32_e32 v2, s2
-; GCN1-NEXT: v_mov_b32_e32 v3, s3
-; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v0, s2
+; GCN1-NEXT: v_mov_b32_e32 v1, s3
+; GCN1-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_min_i64_ret_addr64_offset:
@@ -7192,33 +7102,31 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GCN2-NEXT: s_mov_b64 s[6:7], 0
+; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s5
+; GCN2-NEXT: v_mov_b32_e32 v5, s4
; GCN2-NEXT: .LBB104_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s5
-; GCN2-NEXT: v_mov_b32_e32 v6, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s1
-; GCN2-NEXT: v_mov_b32_e32 v4, s0
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_mov_b32_e32 v9, v3
+; GCN2-NEXT: v_mov_b32_e32 v8, v2
+; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB104_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN2-NEXT: v_mov_b32_e32 v2, s2
-; GCN2-NEXT: v_mov_b32_e32 v3, s3
-; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v0, s2
+; GCN2-NEXT: v_mov_b32_e32 v1, s3
+; GCN2-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_min_i64_ret_addr64_offset:
@@ -7230,33 +7138,31 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN3-NEXT: s_addc_u32 s1, s1, s7
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
-; GCN3-NEXT: s_mov_b64 s[6:7], 0
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s5
+; GCN3-NEXT: v_mov_b32_e32 v5, s4
; GCN3-NEXT: .LBB104_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s5
-; GCN3-NEXT: v_mov_b32_e32 v6, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s1
-; GCN3-NEXT: v_mov_b32_e32 v4, s0
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_mov_b32_e32 v9, v3
+; GCN3-NEXT: v_mov_b32_e32 v8, v2
+; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] offset:32 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB104_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN3-NEXT: v_mov_b32_e32 v2, s2
-; GCN3-NEXT: v_mov_b32_e32 v3, s3
-; GCN3-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v0, s2
+; GCN3-NEXT: v_mov_b32_e32 v1, s3
+; GCN3-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr i64, ptr %out, i64 %index
@@ -7275,16 +7181,16 @@ define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v5, s1
+; GCN1-NEXT: v_mov_b32_e32 v6, s3
+; GCN1-NEXT: v_mov_b32_e32 v7, s2
+; GCN1-NEXT: v_mov_b32_e32 v4, s0
; GCN1-NEXT: .LBB105_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s3
-; GCN1-NEXT: v_mov_b32_e32 v6, s2
-; GCN1-NEXT: v_mov_b32_e32 v5, s1
-; GCN1-NEXT: v_mov_b32_e32 v4, s0
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -7306,16 +7212,16 @@ define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v5, s1
+; GCN2-NEXT: v_mov_b32_e32 v6, s3
+; GCN2-NEXT: v_mov_b32_e32 v7, s2
+; GCN2-NEXT: v_mov_b32_e32 v4, s0
; GCN2-NEXT: .LBB105_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s3
-; GCN2-NEXT: v_mov_b32_e32 v6, s2
-; GCN2-NEXT: v_mov_b32_e32 v5, s1
-; GCN2-NEXT: v_mov_b32_e32 v4, s0
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -7337,16 +7243,16 @@ define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v5, s1
+; GCN3-NEXT: v_mov_b32_e32 v6, s3
+; GCN3-NEXT: v_mov_b32_e32 v7, s2
+; GCN3-NEXT: v_mov_b32_e32 v4, s0
; GCN3-NEXT: .LBB105_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s3
-; GCN3-NEXT: v_mov_b32_e32 v6, s2
-; GCN3-NEXT: v_mov_b32_e32 v5, s1
-; GCN3-NEXT: v_mov_b32_e32 v4, s0
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -7374,33 +7280,31 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN1-NEXT: s_addc_u32 s1, s1, s7
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GCN1-NEXT: s_mov_b64 s[6:7], 0
+; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s5
+; GCN1-NEXT: v_mov_b32_e32 v5, s4
; GCN1-NEXT: .LBB106_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s5
-; GCN1-NEXT: v_mov_b32_e32 v6, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s1
-; GCN1-NEXT: v_mov_b32_e32 v4, s0
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_mov_b32_e32 v9, v3
+; GCN1-NEXT: v_mov_b32_e32 v8, v2
+; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN1-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB106_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN1-NEXT: v_mov_b32_e32 v2, s2
-; GCN1-NEXT: v_mov_b32_e32 v3, s3
-; GCN1-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v0, s2
+; GCN1-NEXT: v_mov_b32_e32 v1, s3
+; GCN1-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_min_i64_ret_addr64:
@@ -7412,33 +7316,31 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN2-NEXT: s_addc_u32 s1, s1, s7
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GCN2-NEXT: s_mov_b64 s[6:7], 0
+; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s5
+; GCN2-NEXT: v_mov_b32_e32 v5, s4
; GCN2-NEXT: .LBB106_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s5
-; GCN2-NEXT: v_mov_b32_e32 v6, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s1
-; GCN2-NEXT: v_mov_b32_e32 v4, s0
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_mov_b32_e32 v9, v3
+; GCN2-NEXT: v_mov_b32_e32 v8, v2
+; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN2-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB106_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN2-NEXT: v_mov_b32_e32 v2, s2
-; GCN2-NEXT: v_mov_b32_e32 v3, s3
-; GCN2-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v0, s2
+; GCN2-NEXT: v_mov_b32_e32 v1, s3
+; GCN2-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_min_i64_ret_addr64:
@@ -7450,33 +7352,31 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN3-NEXT: s_addc_u32 s1, s1, s7
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GCN3-NEXT: s_mov_b64 s[6:7], 0
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s5
+; GCN3-NEXT: v_mov_b32_e32 v5, s4
; GCN3-NEXT: .LBB106_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s5
-; GCN3-NEXT: v_mov_b32_e32 v6, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s1
-; GCN3-NEXT: v_mov_b32_e32 v4, s0
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_mov_b32_e32 v9, v3
+; GCN3-NEXT: v_mov_b32_e32 v8, v2
+; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GCN3-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB106_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN3-NEXT: v_mov_b32_e32 v2, s2
-; GCN3-NEXT: v_mov_b32_e32 v3, s3
-; GCN3-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v0, s2
+; GCN3-NEXT: v_mov_b32_e32 v1, s3
+; GCN3-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr i64, ptr %out, i64 %index
diff --git a/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
index fdc8c908ded549c..ee5e0e99510577b 100644
--- a/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
@@ -1534,16 +1534,16 @@ main_body:
define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat(ptr %ptr) #1 {
; GFX90A-LABEL: flat_atomic_fadd_f64_noret_pat:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX90A-NEXT: s_mov_b64 s[2:3], 0
+; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX90A-NEXT: s_mov_b64 s[0:1], 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -1551,9 +1551,9 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat(ptr %ptr) #1 {
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX90A-NEXT: s_cbranch_execnz .LBB53_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_endpgm
@@ -1609,16 +1609,16 @@ main_body:
define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_system(ptr %ptr) #1 {
; GFX90A-LABEL: flat_atomic_fadd_f64_noret_pat_system:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX90A-NEXT: s_mov_b64 s[2:3], 0
+; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX90A-NEXT: s_mov_b64 s[0:1], 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -1627,9 +1627,9 @@ define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_system(ptr %ptr) #1 {
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX90A-NEXT: s_cbranch_execnz .LBB55_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_endpgm
@@ -1813,24 +1813,24 @@ main_body:
define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_agent_safe(ptr %ptr) {
; GFX90A-LABEL: flat_atomic_fadd_f64_noret_pat_agent_safe:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX90A-NEXT: s_mov_b64 s[2:3], 0
+; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
+; GFX90A-NEXT: s_mov_b64 s[0:1], 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX90A-NEXT: s_cbranch_execnz .LBB61_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_endpgm
@@ -2104,17 +2104,17 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrsp
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v0, s2
; GFX90A-NEXT: ds_read_b64 v[0:1], v0
+; GFX90A-NEXT: v_mov_b32_e32 v2, s2
; GFX90A-NEXT: .LBB72_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_add_f64 v[2:3], v[0:1], 4.0
-; GFX90A-NEXT: v_mov_b32_e32 v4, s2
+; GFX90A-NEXT: v_add_f64 v[4:5], v[0:1], 4.0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: ds_cmpst_rtn_b64 v[2:3], v4, v[0:1], v[2:3]
+; GFX90A-NEXT: ds_cmpst_rtn_b64 v[4:5], v2, v[0:1], v[4:5]
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[0:1]
+; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[0:1]
; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX90A-NEXT: s_cbranch_execnz .LBB72_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2127,17 +2127,17 @@ define amdgpu_kernel void @local_atomic_fadd_f64_noret_pat_flush_safe(ptr addrsp
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
; GFX940-NEXT: v_mov_b32_e32 v0, s2
; GFX940-NEXT: ds_read_b64 v[0:1], v0
+; GFX940-NEXT: v_mov_b32_e32 v2, s2
; GFX940-NEXT: .LBB72_1: ; %atomicrmw.start
; GFX940-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
-; GFX940-NEXT: v_add_f64 v[2:3], v[0:1], 4.0
-; GFX940-NEXT: v_mov_b32_e32 v4, s2
+; GFX940-NEXT: v_add_f64 v[4:5], v[0:1], 4.0
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
-; GFX940-NEXT: ds_cmpst_rtn_b64 v[2:3], v4, v[0:1], v[2:3]
+; GFX940-NEXT: ds_cmpst_rtn_b64 v[4:5], v2, v[0:1], v[4:5]
; GFX940-NEXT: s_waitcnt lgkmcnt(0)
-; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[0:1]
+; GFX940-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[0:1]
; GFX940-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX940-NEXT: v_mov_b64_e32 v[0:1], v[2:3]
+; GFX940-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX940-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX940-NEXT: s_cbranch_execnz .LBB72_1
; GFX940-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/global-load-saddr-to-vaddr.ll b/llvm/test/CodeGen/AMDGPU/global-load-saddr-to-vaddr.ll
index 8dd2d8a03037000..a2fdeeea0707f78 100644
--- a/llvm/test/CodeGen/AMDGPU/global-load-saddr-to-vaddr.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-load-saddr-to-vaddr.ll
@@ -56,20 +56,19 @@ define amdgpu_kernel void @test_move_load_address_to_vgpr_d16_hi(ptr addrspace(1
; GCN: ; %bb.0: ; %bb
; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; GCN-NEXT: v_mov_b32_e32 v1, 0
-; GCN-NEXT: s_movk_i32 s2, 0x100
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: global_load_ushort v0, v1, s[0:1] glc
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v2, s1
+; GCN-NEXT: s_movk_i32 s1, 0x100
; GCN-NEXT: .LBB1_1: ; %bb3
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
+; GCN-NEXT: v_lshlrev_b64 v[3:4], 1, v[0:1]
+; GCN-NEXT: v_add_co_u32_e32 v3, vcc, s0, v3
+; GCN-NEXT: v_addc_co_u32_e32 v4, vcc, v2, v4, vcc
+; GCN-NEXT: global_load_short_d16_hi v0, v[3:4], off glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_lshlrev_b64 v[2:3], 1, v[0:1]
-; GCN-NEXT: v_mov_b32_e32 v0, s1
-; GCN-NEXT: v_add_co_u32_e32 v2, vcc, s0, v2
-; GCN-NEXT: v_addc_co_u32_e32 v3, vcc, v0, v3, vcc
-; GCN-NEXT: global_load_short_d16_hi v0, v[2:3], off glc
-; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, s2, v0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, s1, v0
; GCN-NEXT: s_cbranch_vccz .LBB1_1
; GCN-NEXT: ; %bb.2: ; %bb2
; GCN-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
index a6d56df00c8627f..c83fb79fde63d12 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
@@ -2773,22 +2773,20 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_scalar(ptr addrspace(1) inr
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
-; VI-NEXT: flat_load_dword v1, v[0:1]
+; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB44_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_and_b32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: v_not_b32_e32 v0, v0
+; VI-NEXT: v_and_b32_e32 v2, s6, v3
+; VI-NEXT: v_not_b32_e32 v2, v2
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; VI-NEXT: v_mov_b32_e32 v1, v0
+; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB44_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2871,26 +2869,24 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_offset_scalar(ptr addrspace
; VI-NEXT: s_addc_u32 s35, s5, 0
; VI-NEXT: v_mov_b32_e32 v0, s34
; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v1, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: flat_load_dword v3, v[0:1]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB45_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_and_b32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: v_not_b32_e32 v0, v0
+; VI-NEXT: v_and_b32_e32 v2, s6, v3
+; VI-NEXT: v_not_b32_e32 v2, v2
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB45_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_nand_i32_noret_offset_scalar:
@@ -2970,20 +2966,20 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_scalar(ptr addrspace(1) inreg
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dword v0, v[0:1]
+; VI-NEXT: v_mov_b32_e32 v1, s4
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: .LBB46_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_and_b32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: v_not_b32_e32 v0, v0
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_and_b32_e32 v0, s6, v4
+; VI-NEXT: v_not_b32_e32 v3, v0
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB46_1
@@ -3066,28 +3062,26 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 16
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v0, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v1, s34
+; VI-NEXT: v_mov_b32_e32 v2, s35
+; VI-NEXT: flat_load_dword v0, v[1:2]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB47_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_and_b32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: v_not_b32_e32 v0, v0
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_and_b32_e32 v0, s6, v4
+; VI-NEXT: v_not_b32_e32 v3, v0
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB47_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_nand_i32_ret_offset_scalar:
@@ -4219,21 +4213,19 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_scalar(ptr addrspace(1) inre
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
-; VI-NEXT: flat_load_dword v1, v[0:1]
+; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB68_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v2, s4
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_max_i32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v3, s5
+; VI-NEXT: v_max_i32_e32 v2, s6, v3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; VI-NEXT: v_mov_b32_e32 v1, v0
+; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB68_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4314,25 +4306,23 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_offset_scalar(ptr addrspace(
; VI-NEXT: s_addc_u32 s35, s5, 0
; VI-NEXT: v_mov_b32_e32 v0, s34
; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v1, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: flat_load_dword v3, v[0:1]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB69_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v2, s34
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_max_i32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v3, s35
+; VI-NEXT: v_max_i32_e32 v2, s6, v3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB69_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_max_i32_noret_offset_scalar:
@@ -4410,19 +4400,19 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_scalar(ptr addrspace(1) inreg %
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dword v0, v[0:1]
+; VI-NEXT: v_mov_b32_e32 v1, s4
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: .LBB70_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: v_max_i32_e32 v0, s6, v1
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_max_i32_e32 v3, s6, v4
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB70_1
@@ -4503,27 +4493,25 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 16
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v0, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v1, s34
+; VI-NEXT: v_mov_b32_e32 v2, s35
+; VI-NEXT: flat_load_dword v0, v[1:2]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB71_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: v_max_i32_e32 v0, s6, v1
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_max_i32_e32 v3, s6, v4
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB71_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_max_i32_ret_offset_scalar:
@@ -4594,27 +4582,27 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr addrspace(1) %out, i
; VI-NEXT: s_ashr_i32 s5, s3, 31
; VI-NEXT: s_mov_b32 s4, s3
; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; VI-NEXT: s_add_u32 s0, s0, s4
-; VI-NEXT: s_addc_u32 s1, s1, s5
-; VI-NEXT: s_load_dword s3, s[0:1], 0x10
-; VI-NEXT: s_add_u32 s0, s0, 16
-; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_add_u32 s4, s0, s4
+; VI-NEXT: s_addc_u32 s5, s1, s5
+; VI-NEXT: s_load_dword s3, s[4:5], 0x10
+; VI-NEXT: s_add_u32 s4, s4, 16
+; VI-NEXT: s_addc_u32 s5, s5, 0
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: s_mov_b64 s[0:1], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: v_mov_b32_e32 v3, s3
+; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: .LBB72_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_max_i32_e32 v0, s2, v1
-; VI-NEXT: v_mov_b32_e32 v2, s0
+; VI-NEXT: v_max_i32_e32 v2, s2, v3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB72_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_endpgm
@@ -4703,33 +4691,33 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr addrspace(1) %ou
; VI-NEXT: s_ashr_i32 s7, s5, 31
; VI-NEXT: s_mov_b32 s6, s5
; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 2
-; VI-NEXT: s_add_u32 s0, s0, s6
-; VI-NEXT: s_addc_u32 s1, s1, s7
-; VI-NEXT: s_load_dword s5, s[0:1], 0x10
-; VI-NEXT: s_add_u32 s0, s0, 16
-; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: s_mov_b64 s[6:7], 0
+; VI-NEXT: s_add_u32 s6, s0, s6
+; VI-NEXT: s_addc_u32 s7, s1, s7
+; VI-NEXT: s_load_dword s5, s[6:7], 0x10
+; VI-NEXT: s_add_u32 s6, s6, 16
+; VI-NEXT: s_addc_u32 s7, s7, 0
+; VI-NEXT: v_mov_b32_e32 v0, s6
+; VI-NEXT: s_mov_b64 s[0:1], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s5
+; VI-NEXT: v_mov_b32_e32 v2, s5
+; VI-NEXT: v_mov_b32_e32 v1, s7
; VI-NEXT: .LBB73_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: v_max_i32_e32 v0, s4, v1
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: v_max_i32_e32 v2, s4, v3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; VI-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB73_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[6:7]
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_mov_b32_e32 v2, s3
-; VI-NEXT: flat_store_dword v[1:2], v0
+; VI-NEXT: s_or_b64 exec, exec, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_max_i32_ret_addr64_offset:
@@ -4813,25 +4801,25 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr addrspace(1) %out, i32 %in,
; VI-NEXT: s_ashr_i32 s5, s3, 31
; VI-NEXT: s_mov_b32 s4, s3
; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; VI-NEXT: s_add_u32 s0, s0, s4
-; VI-NEXT: s_addc_u32 s1, s1, s5
-; VI-NEXT: s_load_dword s3, s[0:1], 0x0
-; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_add_u32 s4, s0, s4
+; VI-NEXT: s_addc_u32 s5, s1, s5
+; VI-NEXT: s_load_dword s3, s[4:5], 0x0
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: v_mov_b32_e32 v3, s3
; VI-NEXT: .LBB74_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_max_i32_e32 v0, s2, v1
-; VI-NEXT: v_mov_b32_e32 v2, s0
+; VI-NEXT: v_max_i32_e32 v2, s2, v3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB74_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_endpgm
@@ -4919,31 +4907,31 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr addrspace(1) %out, ptr
; VI-NEXT: s_ashr_i32 s7, s5, 31
; VI-NEXT: s_mov_b32 s6, s5
; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 2
-; VI-NEXT: s_add_u32 s0, s0, s6
-; VI-NEXT: s_addc_u32 s1, s1, s7
-; VI-NEXT: s_load_dword s5, s[0:1], 0x0
-; VI-NEXT: s_mov_b64 s[6:7], 0
+; VI-NEXT: s_add_u32 s6, s0, s6
+; VI-NEXT: s_addc_u32 s7, s1, s7
+; VI-NEXT: s_load_dword s5, s[6:7], 0x0
+; VI-NEXT: v_mov_b32_e32 v0, s6
+; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: v_mov_b32_e32 v1, s7
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s5
+; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: .LBB75_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: v_max_i32_e32 v0, s4, v1
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: v_max_i32_e32 v2, s4, v3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; VI-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB75_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[6:7]
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_mov_b32_e32 v2, s3
-; VI-NEXT: flat_store_dword v[1:2], v0
+; VI-NEXT: s_or_b64 exec, exec, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_max_i32_ret_addr64:
@@ -5358,21 +5346,19 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_scalar(ptr addrspace(1) inr
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
-; VI-NEXT: flat_load_dword v1, v[0:1]
+; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB80_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v2, s4
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_max_u32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v3, s5
+; VI-NEXT: v_max_u32_e32 v2, s6, v3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; VI-NEXT: v_mov_b32_e32 v1, v0
+; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB80_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -5453,25 +5439,23 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_offset_scalar(ptr addrspace
; VI-NEXT: s_addc_u32 s35, s5, 0
; VI-NEXT: v_mov_b32_e32 v0, s34
; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v1, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: flat_load_dword v3, v[0:1]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB81_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v2, s34
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_max_u32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v3, s35
+; VI-NEXT: v_max_u32_e32 v2, s6, v3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB81_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_umax_i32_noret_offset_scalar:
@@ -5549,19 +5533,19 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_scalar(ptr addrspace(1) inreg
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dword v0, v[0:1]
+; VI-NEXT: v_mov_b32_e32 v1, s4
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: .LBB82_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: v_max_u32_e32 v0, s6, v1
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_max_u32_e32 v3, s6, v4
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB82_1
@@ -5642,27 +5626,25 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 16
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v0, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v1, s34
+; VI-NEXT: v_mov_b32_e32 v2, s35
+; VI-NEXT: flat_load_dword v0, v[1:2]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB83_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: v_max_u32_e32 v0, s6, v1
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_max_u32_e32 v3, s6, v4
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB83_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_umax_i32_ret_offset_scalar:
@@ -5733,27 +5715,27 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr addrspace(1) %out,
; VI-NEXT: s_ashr_i32 s5, s3, 31
; VI-NEXT: s_mov_b32 s4, s3
; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; VI-NEXT: s_add_u32 s0, s0, s4
-; VI-NEXT: s_addc_u32 s1, s1, s5
-; VI-NEXT: s_load_dword s3, s[0:1], 0x10
-; VI-NEXT: s_add_u32 s0, s0, 16
-; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_add_u32 s4, s0, s4
+; VI-NEXT: s_addc_u32 s5, s1, s5
+; VI-NEXT: s_load_dword s3, s[4:5], 0x10
+; VI-NEXT: s_add_u32 s4, s4, 16
+; VI-NEXT: s_addc_u32 s5, s5, 0
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: s_mov_b64 s[0:1], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: v_mov_b32_e32 v3, s3
+; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: .LBB84_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_max_u32_e32 v0, s2, v1
-; VI-NEXT: v_mov_b32_e32 v2, s0
+; VI-NEXT: v_max_u32_e32 v2, s2, v3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB84_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_endpgm
@@ -5842,33 +5824,33 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr addrspace(1) %o
; VI-NEXT: s_ashr_i32 s7, s5, 31
; VI-NEXT: s_mov_b32 s6, s5
; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 2
-; VI-NEXT: s_add_u32 s0, s0, s6
-; VI-NEXT: s_addc_u32 s1, s1, s7
-; VI-NEXT: s_load_dword s5, s[0:1], 0x10
-; VI-NEXT: s_add_u32 s0, s0, 16
-; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: s_mov_b64 s[6:7], 0
+; VI-NEXT: s_add_u32 s6, s0, s6
+; VI-NEXT: s_addc_u32 s7, s1, s7
+; VI-NEXT: s_load_dword s5, s[6:7], 0x10
+; VI-NEXT: s_add_u32 s6, s6, 16
+; VI-NEXT: s_addc_u32 s7, s7, 0
+; VI-NEXT: v_mov_b32_e32 v0, s6
+; VI-NEXT: s_mov_b64 s[0:1], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s5
+; VI-NEXT: v_mov_b32_e32 v2, s5
+; VI-NEXT: v_mov_b32_e32 v1, s7
; VI-NEXT: .LBB85_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: v_max_u32_e32 v0, s4, v1
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: v_max_u32_e32 v2, s4, v3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; VI-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB85_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[6:7]
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_mov_b32_e32 v2, s3
-; VI-NEXT: flat_store_dword v[1:2], v0
+; VI-NEXT: s_or_b64 exec, exec, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_umax_i32_ret_addr64_offset:
@@ -5960,31 +5942,31 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr addrspace(1) %out, ptr
; VI-NEXT: s_ashr_i32 s7, s5, 31
; VI-NEXT: s_mov_b32 s6, s5
; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 2
-; VI-NEXT: s_add_u32 s0, s0, s6
-; VI-NEXT: s_addc_u32 s1, s1, s7
-; VI-NEXT: s_load_dword s5, s[0:1], 0x0
-; VI-NEXT: s_mov_b64 s[6:7], 0
+; VI-NEXT: s_add_u32 s6, s0, s6
+; VI-NEXT: s_addc_u32 s7, s1, s7
+; VI-NEXT: s_load_dword s5, s[6:7], 0x0
+; VI-NEXT: v_mov_b32_e32 v0, s6
+; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: v_mov_b32_e32 v1, s7
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s5
+; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: .LBB86_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: v_max_u32_e32 v0, s4, v1
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: v_max_u32_e32 v2, s4, v3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; VI-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB86_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[6:7]
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_mov_b32_e32 v2, s3
-; VI-NEXT: flat_store_dword v[1:2], v0
+; VI-NEXT: s_or_b64 exec, exec, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_umax_i32_ret_addr64:
@@ -6399,21 +6381,19 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_scalar(ptr addrspace(1) inr
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
-; VI-NEXT: flat_load_dword v1, v[0:1]
+; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB91_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v2, s4
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_min_u32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v3, s5
+; VI-NEXT: v_min_u32_e32 v2, s6, v3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; VI-NEXT: v_mov_b32_e32 v1, v0
+; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB91_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6494,25 +6474,23 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_offset_scalar(ptr addrspace
; VI-NEXT: s_addc_u32 s35, s5, 0
; VI-NEXT: v_mov_b32_e32 v0, s34
; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v1, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: flat_load_dword v3, v[0:1]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB92_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v2, s34
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_min_u32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v3, s35
+; VI-NEXT: v_min_u32_e32 v2, s6, v3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB92_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_umin_i32_noret_offset_scalar:
@@ -6590,19 +6568,19 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_scalar(ptr addrspace(1) inreg
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dword v0, v[0:1]
+; VI-NEXT: v_mov_b32_e32 v1, s4
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: .LBB93_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: v_min_u32_e32 v0, s6, v1
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_min_u32_e32 v3, s6, v4
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB93_1
@@ -6683,27 +6661,25 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 16
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v0, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v1, s34
+; VI-NEXT: v_mov_b32_e32 v2, s35
+; VI-NEXT: flat_load_dword v0, v[1:2]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB94_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: v_min_u32_e32 v0, s6, v1
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_min_u32_e32 v3, s6, v4
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB94_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_umin_i32_ret_offset_scalar:
@@ -7106,21 +7082,19 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_scalar(ptr addrspace(1) inre
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
-; VI-NEXT: flat_load_dword v1, v[0:1]
+; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB99_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v2, s4
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_min_i32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v3, s5
+; VI-NEXT: v_min_i32_e32 v2, s6, v3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; VI-NEXT: v_mov_b32_e32 v1, v0
+; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB99_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7201,25 +7175,23 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_offset_scalar(ptr addrspace(
; VI-NEXT: s_addc_u32 s35, s5, 0
; VI-NEXT: v_mov_b32_e32 v0, s34
; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v1, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: flat_load_dword v3, v[0:1]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB100_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v2, s34
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_min_i32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v3, s35
+; VI-NEXT: v_min_i32_e32 v2, s6, v3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB100_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_min_i32_noret_offset_scalar:
@@ -7297,19 +7269,19 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_scalar(ptr addrspace(1) inreg %
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dword v0, v[0:1]
+; VI-NEXT: v_mov_b32_e32 v1, s4
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: .LBB101_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: v_min_i32_e32 v0, s6, v1
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_min_i32_e32 v3, s6, v4
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB101_1
@@ -7390,27 +7362,25 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 16
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v0, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v1, s34
+; VI-NEXT: v_mov_b32_e32 v2, s35
+; VI-NEXT: flat_load_dword v0, v[1:2]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB102_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: v_min_i32_e32 v0, s6, v1
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_min_i32_e32 v3, s6, v4
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB102_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_min_i32_ret_offset_scalar:
@@ -7481,27 +7451,27 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr addrspace(1) %out, i
; VI-NEXT: s_ashr_i32 s5, s3, 31
; VI-NEXT: s_mov_b32 s4, s3
; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; VI-NEXT: s_add_u32 s0, s0, s4
-; VI-NEXT: s_addc_u32 s1, s1, s5
-; VI-NEXT: s_load_dword s3, s[0:1], 0x10
-; VI-NEXT: s_add_u32 s0, s0, 16
-; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_add_u32 s4, s0, s4
+; VI-NEXT: s_addc_u32 s5, s1, s5
+; VI-NEXT: s_load_dword s3, s[4:5], 0x10
+; VI-NEXT: s_add_u32 s4, s4, 16
+; VI-NEXT: s_addc_u32 s5, s5, 0
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: s_mov_b64 s[0:1], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: v_mov_b32_e32 v3, s3
+; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: .LBB103_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_min_i32_e32 v0, s2, v1
-; VI-NEXT: v_mov_b32_e32 v2, s0
+; VI-NEXT: v_min_i32_e32 v2, s2, v3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB103_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_endpgm
@@ -7590,33 +7560,33 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr addrspace(1) %ou
; VI-NEXT: s_ashr_i32 s7, s5, 31
; VI-NEXT: s_mov_b32 s6, s5
; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 2
-; VI-NEXT: s_add_u32 s0, s0, s6
-; VI-NEXT: s_addc_u32 s1, s1, s7
-; VI-NEXT: s_load_dword s5, s[0:1], 0x10
-; VI-NEXT: s_add_u32 s0, s0, 16
-; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: s_mov_b64 s[6:7], 0
+; VI-NEXT: s_add_u32 s6, s0, s6
+; VI-NEXT: s_addc_u32 s7, s1, s7
+; VI-NEXT: s_load_dword s5, s[6:7], 0x10
+; VI-NEXT: s_add_u32 s6, s6, 16
+; VI-NEXT: s_addc_u32 s7, s7, 0
+; VI-NEXT: v_mov_b32_e32 v0, s6
+; VI-NEXT: s_mov_b64 s[0:1], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s5
+; VI-NEXT: v_mov_b32_e32 v2, s5
+; VI-NEXT: v_mov_b32_e32 v1, s7
; VI-NEXT: .LBB104_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: v_min_i32_e32 v0, s4, v1
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: v_min_i32_e32 v2, s4, v3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; VI-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB104_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[6:7]
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_mov_b32_e32 v2, s3
-; VI-NEXT: flat_store_dword v[1:2], v0
+; VI-NEXT: s_or_b64 exec, exec, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_min_i32_ret_addr64_offset:
@@ -7691,25 +7661,25 @@ define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) {
;
; VI-LABEL: atomic_min_i32:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
-; VI-NEXT: s_load_dword s4, s[0:1], 0x2c
+; VI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24
+; VI-NEXT: s_load_dword s2, s[0:1], 0x2c
; VI-NEXT: s_mov_b64 s[0:1], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_load_dword s5, s[2:3], 0x0
-; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_load_dword s3, s[4:5], 0x0
+; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v3, s3
; VI-NEXT: .LBB105_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_min_i32_e32 v0, s4, v1
-; VI-NEXT: v_mov_b32_e32 v3, s3
+; VI-NEXT: v_min_i32_e32 v2, s2, v3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; VI-NEXT: v_mov_b32_e32 v1, v0
+; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB105_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7793,31 +7763,31 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr addrspace(1) %out, ptr
; VI-NEXT: s_ashr_i32 s7, s5, 31
; VI-NEXT: s_mov_b32 s6, s5
; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 2
-; VI-NEXT: s_add_u32 s0, s0, s6
-; VI-NEXT: s_addc_u32 s1, s1, s7
-; VI-NEXT: s_load_dword s5, s[0:1], 0x0
-; VI-NEXT: s_mov_b64 s[6:7], 0
+; VI-NEXT: s_add_u32 s6, s0, s6
+; VI-NEXT: s_addc_u32 s7, s1, s7
+; VI-NEXT: s_load_dword s5, s[6:7], 0x0
+; VI-NEXT: v_mov_b32_e32 v0, s6
+; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: v_mov_b32_e32 v1, s7
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s5
+; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: .LBB106_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: v_min_i32_e32 v0, s4, v1
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: v_min_i32_e32 v2, s4, v3
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; VI-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB106_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[6:7]
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_mov_b32_e32 v2, s3
-; VI-NEXT: flat_store_dword v[1:2], v0
+; VI-NEXT: s_or_b64 exec, exec, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_min_i32_ret_addr64:
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_i64_system.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_i64_system.ll
index 005cfe73671bd23..5a827584aa1062a 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_i64_system.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_i64_system.ll
@@ -2903,14 +2903,14 @@ define amdgpu_gfx void @global_atomic_nand_i64_noret_scalar(ptr addrspace(1) inr
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: v_mov_b32_e32 v4, s4
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: .LBB44_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_and_b32_e32 v0, s7, v3
; VI-NEXT: v_and_b32_e32 v6, s6, v2
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: v_not_b32_e32 v1, v0
; VI-NEXT: v_not_b32_e32 v0, v6
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -3010,17 +3010,15 @@ define amdgpu_gfx void @global_atomic_nand_i64_noret_offset_scalar(ptr addrspace
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v4, s34
+; VI-NEXT: v_mov_b32_e32 v5, s35
+; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB45_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_and_b32_e32 v0, s7, v3
; VI-NEXT: v_and_b32_e32 v6, s6, v2
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
; VI-NEXT: v_not_b32_e32 v1, v0
; VI-NEXT: v_not_b32_e32 v0, v6
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -3029,12 +3027,12 @@ define amdgpu_gfx void @global_atomic_nand_i64_noret_offset_scalar(ptr addrspace
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB45_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_nand_i64_noret_offset_scalar:
@@ -3124,23 +3122,23 @@ define amdgpu_gfx i64 @global_atomic_nand_i64_ret_scalar(ptr addrspace(1) inreg
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT: v_mov_b32_e32 v2, s4
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: .LBB46_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_and_b32_e32 v0, s7, v3
-; VI-NEXT: v_and_b32_e32 v6, s6, v2
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_not_b32_e32 v1, v0
-; VI-NEXT: v_not_b32_e32 v0, v6
+; VI-NEXT: v_mov_b32_e32 v7, v1
+; VI-NEXT: v_mov_b32_e32 v6, v0
+; VI-NEXT: v_and_b32_e32 v0, s7, v7
+; VI-NEXT: v_and_b32_e32 v1, s6, v6
+; VI-NEXT: v_not_b32_e32 v5, v0
+; VI-NEXT: v_not_b32_e32 v4, v1
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB46_1
@@ -3233,31 +3231,29 @@ define amdgpu_gfx i64 @global_atomic_nand_i64_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v2, s34
+; VI-NEXT: v_mov_b32_e32 v3, s35
+; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB47_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_and_b32_e32 v0, s7, v3
-; VI-NEXT: v_and_b32_e32 v6, s6, v2
-; VI-NEXT: v_mov_b32_e32 v5, s35
-; VI-NEXT: v_not_b32_e32 v1, v0
-; VI-NEXT: v_not_b32_e32 v0, v6
+; VI-NEXT: v_mov_b32_e32 v7, v1
+; VI-NEXT: v_mov_b32_e32 v6, v0
+; VI-NEXT: v_and_b32_e32 v0, s7, v7
+; VI-NEXT: v_and_b32_e32 v1, s6, v6
+; VI-NEXT: v_not_b32_e32 v5, v0
+; VI-NEXT: v_not_b32_e32 v4, v1
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB47_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_nand_i64_ret_offset_scalar:
@@ -4443,27 +4439,27 @@ define amdgpu_gfx void @global_atomic_max_i64_noret_scalar(ptr addrspace(1) inre
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[3:4], off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v5, s35
+; SI-NEXT: v_mov_b32_e32 v6, s34
; SI-NEXT: .LBB68_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: v_mov_b32_e32 v1, s35
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[34:35], v[3:4]
-; SI-NEXT: v_cndmask_b32_e32 v2, v1, v4, vcc
-; SI-NEXT: v_mov_b32_e32 v1, s34
-; SI-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
+; SI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v8, v4
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
+; SI-NEXT: v_mov_b32_e32 v10, v4
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
; SI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[5:8], off, s[4:7], 0 glc
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[7:10], off, s[4:7], 0 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[5:6], v[3:4]
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[3:4]
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; SI-NEXT: v_mov_b32_e32 v3, v5
-; SI-NEXT: v_mov_b32_e32 v4, v6
+; SI-NEXT: v_mov_b32_e32 v3, v7
+; SI-NEXT: v_mov_b32_e32 v4, v8
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
; SI-NEXT: s_cbranch_execnz .LBB68_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4482,17 +4478,17 @@ define amdgpu_gfx void @global_atomic_max_i64_noret_scalar(ptr addrspace(1) inre
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: v_mov_b32_e32 v4, s4
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v6, s7
+; VI-NEXT: v_mov_b32_e32 v7, s6
+; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: .LBB68_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -4513,14 +4509,14 @@ define amdgpu_gfx void @global_atomic_max_i64_noret_scalar(ptr addrspace(1) inre
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s7
+; GFX9-NEXT: v_mov_b32_e32 v6, s6
; GFX9-NEXT: .LBB68_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -4554,27 +4550,27 @@ define amdgpu_gfx void @global_atomic_max_i64_noret_offset_scalar(ptr addrspace(
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[3:4], off, s[4:7], 0 offset:32
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v5, s35
+; SI-NEXT: v_mov_b32_e32 v6, s34
; SI-NEXT: .LBB69_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: v_mov_b32_e32 v1, s35
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[34:35], v[3:4]
-; SI-NEXT: v_cndmask_b32_e32 v2, v1, v4, vcc
-; SI-NEXT: v_mov_b32_e32 v1, s34
-; SI-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
+; SI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v8, v4
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
+; SI-NEXT: v_mov_b32_e32 v10, v4
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
; SI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[5:8], off, s[4:7], 0 offset:32 glc
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[7:10], off, s[4:7], 0 offset:32 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[5:6], v[3:4]
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[3:4]
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; SI-NEXT: v_mov_b32_e32 v3, v5
-; SI-NEXT: v_mov_b32_e32 v4, v6
+; SI-NEXT: v_mov_b32_e32 v3, v7
+; SI-NEXT: v_mov_b32_e32 v4, v8
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
; SI-NEXT: s_cbranch_execnz .LBB69_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4592,32 +4588,30 @@ define amdgpu_gfx void @global_atomic_max_i64_noret_offset_scalar(ptr addrspace(
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v4, s34
+; VI-NEXT: v_mov_b32_e32 v5, s35
+; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v6, s7
+; VI-NEXT: v_mov_b32_e32 v7, s6
; VI-NEXT: .LBB69_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB69_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_max_i64_noret_offset_scalar:
@@ -4626,14 +4620,14 @@ define amdgpu_gfx void @global_atomic_max_i64_noret_offset_scalar(ptr addrspace(
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s7
+; GFX9-NEXT: v_mov_b32_e32 v6, s6
; GFX9-NEXT: .LBB69_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -4668,17 +4662,17 @@ define amdgpu_gfx i64 @global_atomic_max_i64_ret_scalar(ptr addrspace(1) inreg %
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[3:4], off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v0, s35
+; SI-NEXT: v_mov_b32_e32 v1, s34
; SI-NEXT: .LBB70_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v8, v4
; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v0, s35
; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[34:35], v[7:8]
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_cndmask_b32_e32 v6, v0, v8, vcc
-; SI-NEXT: v_mov_b32_e32 v0, s34
-; SI-NEXT: v_cndmask_b32_e32 v5, v0, v7, vcc
+; SI-NEXT: v_cndmask_b32_e32 v5, v1, v7, vcc
; SI-NEXT: v_mov_b32_e32 v3, v5
; SI-NEXT: v_mov_b32_e32 v4, v6
; SI-NEXT: v_mov_b32_e32 v5, v7
@@ -4709,24 +4703,24 @@ define amdgpu_gfx i64 @global_atomic_max_i64_ret_scalar(ptr addrspace(1) inreg %
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT: v_mov_b32_e32 v2, s4
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v4, s7
+; VI-NEXT: v_mov_b32_e32 v5, s6
+; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: .LBB70_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_mov_b32_e32 v9, v1
+; VI-NEXT: v_mov_b32_e32 v8, v0
+; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB70_1
@@ -4740,21 +4734,21 @@ define amdgpu_gfx i64 @global_atomic_max_i64_ret_scalar(ptr addrspace(1) inreg %
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
+; GFX9-NEXT: v_mov_b32_e32 v4, s6
; GFX9-NEXT: .LBB70_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v7, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB70_1
@@ -4781,17 +4775,17 @@ define amdgpu_gfx i64 @global_atomic_max_i64_ret_offset_scalar(ptr addrspace(1)
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[3:4], off, s[4:7], 0 offset:32
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v0, s35
+; SI-NEXT: v_mov_b32_e32 v1, s34
; SI-NEXT: .LBB71_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v8, v4
; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v0, s35
; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[34:35], v[7:8]
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_cndmask_b32_e32 v6, v0, v8, vcc
-; SI-NEXT: v_mov_b32_e32 v0, s34
-; SI-NEXT: v_cndmask_b32_e32 v5, v0, v7, vcc
+; SI-NEXT: v_cndmask_b32_e32 v5, v1, v7, vcc
; SI-NEXT: v_mov_b32_e32 v3, v5
; SI-NEXT: v_mov_b32_e32 v4, v6
; SI-NEXT: v_mov_b32_e32 v5, v7
@@ -4821,32 +4815,30 @@ define amdgpu_gfx i64 @global_atomic_max_i64_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v2, s34
+; VI-NEXT: v_mov_b32_e32 v3, s35
+; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
+; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v4, s7
+; VI-NEXT: v_mov_b32_e32 v5, s6
; VI-NEXT: .LBB71_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_mov_b32_e32 v9, v1
+; VI-NEXT: v_mov_b32_e32 v8, v0
+; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB71_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_max_i64_ret_offset_scalar:
@@ -4855,21 +4847,21 @@ define amdgpu_gfx i64 @global_atomic_max_i64_ret_offset_scalar(ptr addrspace(1)
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
+; GFX9-NEXT: v_mov_b32_e32 v4, s6
; GFX9-NEXT: .LBB71_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v7, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] offset:32 glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB71_1
@@ -4893,30 +4885,30 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr addrspace(1) %out, i
; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x8
; SI-NEXT: s_mov_b64 s[0:1], 0
; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: v_mov_b32_e32 v4, s3
+; SI-NEXT: v_mov_b32_e32 v5, s2
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s8
; SI-NEXT: v_mov_b32_e32 v3, s9
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: .LBB72_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: v_mov_b32_e32 v0, s3
; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_mov_b32_e32 v0, s2
-; SI-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
-; SI-NEXT: v_mov_b32_e32 v4, v0
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
+; SI-NEXT: v_mov_b32_e32 v6, v0
; SI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 offset:32 glc
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 offset:32 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; SI-NEXT: v_mov_b32_e32 v2, v4
-; SI-NEXT: v_mov_b32_e32 v3, v5
+; SI-NEXT: v_mov_b32_e32 v2, v6
+; SI-NEXT: v_mov_b32_e32 v3, v7
; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; SI-NEXT: s_cbranch_execnz .LBB72_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4924,28 +4916,28 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr addrspace(1) %out, i
;
; VI-LABEL: atomic_max_i64_addr64_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
+; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x34
; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], 3
-; VI-NEXT: s_add_u32 s0, s0, s4
-; VI-NEXT: s_addc_u32 s1, s1, s5
+; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
+; VI-NEXT: s_add_u32 s0, s0, s6
+; VI-NEXT: s_addc_u32 s1, s1, s7
; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x20
; VI-NEXT: s_add_u32 s0, s0, 32
; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: v_mov_b32_e32 v5, s1
+; VI-NEXT: v_mov_b32_e32 v6, s3
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s6
+; VI-NEXT: v_mov_b32_e32 v7, s2
; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: v_mov_b32_e32 v4, s0
; VI-NEXT: .LBB72_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s3
-; VI-NEXT: v_mov_b32_e32 v6, s2
-; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -4963,25 +4955,25 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr addrspace(1) %out, i
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[0:1], s[2:3], 3
; GFX9-NEXT: s_add_u32 s0, s4, s0
; GFX9-NEXT: s_addc_u32 s1, s5, s1
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x20
; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s7
+; GFX9-NEXT: v_mov_b32_e32 v5, s6
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s4
; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB72_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] offset:32 glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -5010,17 +5002,17 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr addrspace(1) %ou
; SI-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
; SI-NEXT: s_mov_b64 s[0:1], 0
; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: v_mov_b32_e32 v8, s5
+; SI-NEXT: v_mov_b32_e32 v9, s4
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: s_mov_b32 s10, -1
; SI-NEXT: .LBB73_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: v_mov_b32_e32 v0, s5
; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; SI-NEXT: v_cndmask_b32_e32 v1, v8, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v9, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v7, v3
; SI-NEXT: v_mov_b32_e32 v6, v2
@@ -5048,70 +5040,70 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr addrspace(1) %ou
; VI-LABEL: atomic_max_i64_ret_addr64_offset:
; VI: ; %bb.0: ; %entry
; VI-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24
+; VI-NEXT: s_mov_b64 s[8:9], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; VI-NEXT: s_add_u32 s0, s0, s6
; VI-NEXT: s_addc_u32 s1, s1, s7
-; VI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x20
+; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x20
; VI-NEXT: s_add_u32 s0, s0, 32
; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: s_mov_b64 s[6:7], 0
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v4, s5
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s8
-; VI-NEXT: v_mov_b32_e32 v1, s9
+; VI-NEXT: v_mov_b32_e32 v2, s6
+; VI-NEXT: v_mov_b32_e32 v5, s4
+; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: .LBB73_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s5
-; VI-NEXT: v_mov_b32_e32 v6, s4
-; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_mov_b32_e32 v9, v3
+; VI-NEXT: v_mov_b32_e32 v8, v2
+; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; VI-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; VI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; VI-NEXT: s_andn2_b64 exec, exec, s[8:9]
; VI-NEXT: s_cbranch_execnz .LBB73_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[6:7]
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_or_b64 exec, exec, s[8:9]
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_max_i64_ret_addr64_offset:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; GFX9-NEXT: s_add_u32 s0, s0, s6
; GFX9-NEXT: s_addc_u32 s1, s1, s7
; GFX9-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x20
; GFX9-NEXT: s_mov_b64 s[6:7], 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s5
+; GFX9-NEXT: v_mov_b32_e32 v3, s4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s8
; GFX9-NEXT: v_mov_b32_e32 v1, s9
; GFX9-NEXT: .LBB73_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s5
-; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[0:1] offset:32 glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[5:8], s[0:1] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX9-NEXT: s_cbranch_execnz .LBB73_1
@@ -5140,30 +5132,30 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr addrspace(1) %out, i64 %in,
; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
; SI-NEXT: s_mov_b64 s[0:1], 0
; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: v_mov_b32_e32 v4, s3
+; SI-NEXT: v_mov_b32_e32 v5, s2
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s8
; SI-NEXT: v_mov_b32_e32 v3, s9
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: .LBB74_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: v_mov_b32_e32 v0, s3
; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_mov_b32_e32 v0, s2
-; SI-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
-; SI-NEXT: v_mov_b32_e32 v4, v0
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
+; SI-NEXT: v_mov_b32_e32 v6, v0
; SI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 glc
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; SI-NEXT: v_mov_b32_e32 v2, v4
-; SI-NEXT: v_mov_b32_e32 v3, v5
+; SI-NEXT: v_mov_b32_e32 v2, v6
+; SI-NEXT: v_mov_b32_e32 v3, v7
; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; SI-NEXT: s_cbranch_execnz .LBB74_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -5175,31 +5167,31 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr addrspace(1) %out, i64 %in,
; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], 3
-; VI-NEXT: s_add_u32 s0, s0, s4
-; VI-NEXT: s_addc_u32 s1, s1, s5
-; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_add_u32 s4, s0, s4
+; VI-NEXT: s_addc_u32 s5, s1, s5
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
+; VI-NEXT: v_mov_b32_e32 v4, s4
+; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: v_mov_b32_e32 v6, s3
+; VI-NEXT: v_mov_b32_e32 v7, s2
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s6
; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: .LBB74_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s3
-; VI-NEXT: v_mov_b32_e32 v6, s2
-; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB74_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_endpgm
@@ -5208,25 +5200,25 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr addrspace(1) %out, i64 %in,
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[0:1], s[2:3], 3
; GFX9-NEXT: s_add_u32 s0, s4, s0
; GFX9-NEXT: s_addc_u32 s1, s5, s1
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s7
+; GFX9-NEXT: v_mov_b32_e32 v5, s6
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s4
; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB74_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -5254,17 +5246,17 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr addrspace(1) %out, ptr
; SI-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
; SI-NEXT: s_mov_b64 s[0:1], 0
; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: v_mov_b32_e32 v8, s5
+; SI-NEXT: v_mov_b32_e32 v9, s4
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: s_mov_b32 s10, -1
; SI-NEXT: .LBB75_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: v_mov_b32_e32 v0, s5
; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; SI-NEXT: v_cndmask_b32_e32 v1, v8, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v9, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v7, v3
; SI-NEXT: v_mov_b32_e32 v6, v2
@@ -5294,66 +5286,66 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr addrspace(1) %out, ptr
; VI-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
-; VI-NEXT: s_add_u32 s0, s0, s6
-; VI-NEXT: s_addc_u32 s1, s1, s7
-; VI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
-; VI-NEXT: s_mov_b64 s[6:7], 0
+; VI-NEXT: s_add_u32 s6, s0, s6
+; VI-NEXT: s_addc_u32 s7, s1, s7
+; VI-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0
+; VI-NEXT: v_mov_b32_e32 v0, s6
+; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: v_mov_b32_e32 v4, s5
+; VI-NEXT: v_mov_b32_e32 v5, s4
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s8
-; VI-NEXT: v_mov_b32_e32 v1, s9
+; VI-NEXT: v_mov_b32_e32 v2, s8
+; VI-NEXT: v_mov_b32_e32 v3, s9
+; VI-NEXT: v_mov_b32_e32 v1, s7
; VI-NEXT: .LBB75_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s5
-; VI-NEXT: v_mov_b32_e32 v6, s4
-; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_mov_b32_e32 v9, v3
+; VI-NEXT: v_mov_b32_e32 v8, v2
+; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; VI-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB75_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[6:7]
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_or_b64 exec, exec, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_max_i64_ret_addr64:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; GFX9-NEXT: s_add_u32 s0, s0, s6
; GFX9-NEXT: s_addc_u32 s1, s1, s7
; GFX9-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
; GFX9-NEXT: s_mov_b64 s[6:7], 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s5
+; GFX9-NEXT: v_mov_b32_e32 v3, s4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s8
; GFX9-NEXT: v_mov_b32_e32 v1, s9
; GFX9-NEXT: .LBB75_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s5
-; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[0:1] glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[5:8], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX9-NEXT: s_cbranch_execnz .LBB75_1
@@ -5765,27 +5757,27 @@ define amdgpu_gfx void @global_atomic_umax_i64_noret_scalar(ptr addrspace(1) inr
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[3:4], off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v5, s35
+; SI-NEXT: v_mov_b32_e32 v6, s34
; SI-NEXT: .LBB80_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: v_mov_b32_e32 v1, s35
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[34:35], v[3:4]
-; SI-NEXT: v_cndmask_b32_e32 v2, v1, v4, vcc
-; SI-NEXT: v_mov_b32_e32 v1, s34
-; SI-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
+; SI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v8, v4
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
+; SI-NEXT: v_mov_b32_e32 v10, v4
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
; SI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[5:8], off, s[4:7], 0 glc
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[7:10], off, s[4:7], 0 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[5:6], v[3:4]
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[3:4]
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; SI-NEXT: v_mov_b32_e32 v3, v5
-; SI-NEXT: v_mov_b32_e32 v4, v6
+; SI-NEXT: v_mov_b32_e32 v3, v7
+; SI-NEXT: v_mov_b32_e32 v4, v8
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
; SI-NEXT: s_cbranch_execnz .LBB80_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -5804,17 +5796,17 @@ define amdgpu_gfx void @global_atomic_umax_i64_noret_scalar(ptr addrspace(1) inr
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: v_mov_b32_e32 v4, s4
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v6, s7
+; VI-NEXT: v_mov_b32_e32 v7, s6
+; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: .LBB80_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -5835,14 +5827,14 @@ define amdgpu_gfx void @global_atomic_umax_i64_noret_scalar(ptr addrspace(1) inr
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s7
+; GFX9-NEXT: v_mov_b32_e32 v6, s6
; GFX9-NEXT: .LBB80_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -5876,27 +5868,27 @@ define amdgpu_gfx void @global_atomic_umax_i64_noret_offset_scalar(ptr addrspace
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[3:4], off, s[4:7], 0 offset:32
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v5, s35
+; SI-NEXT: v_mov_b32_e32 v6, s34
; SI-NEXT: .LBB81_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: v_mov_b32_e32 v1, s35
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[34:35], v[3:4]
-; SI-NEXT: v_cndmask_b32_e32 v2, v1, v4, vcc
-; SI-NEXT: v_mov_b32_e32 v1, s34
-; SI-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
+; SI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v8, v4
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
+; SI-NEXT: v_mov_b32_e32 v10, v4
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
; SI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[5:8], off, s[4:7], 0 offset:32 glc
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[7:10], off, s[4:7], 0 offset:32 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[5:6], v[3:4]
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[3:4]
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; SI-NEXT: v_mov_b32_e32 v3, v5
-; SI-NEXT: v_mov_b32_e32 v4, v6
+; SI-NEXT: v_mov_b32_e32 v3, v7
+; SI-NEXT: v_mov_b32_e32 v4, v8
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
; SI-NEXT: s_cbranch_execnz .LBB81_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -5914,32 +5906,30 @@ define amdgpu_gfx void @global_atomic_umax_i64_noret_offset_scalar(ptr addrspace
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v4, s34
+; VI-NEXT: v_mov_b32_e32 v5, s35
+; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v6, s7
+; VI-NEXT: v_mov_b32_e32 v7, s6
; VI-NEXT: .LBB81_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB81_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_umax_i64_noret_offset_scalar:
@@ -5948,14 +5938,14 @@ define amdgpu_gfx void @global_atomic_umax_i64_noret_offset_scalar(ptr addrspace
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s7
+; GFX9-NEXT: v_mov_b32_e32 v6, s6
; GFX9-NEXT: .LBB81_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -5990,17 +5980,17 @@ define amdgpu_gfx i64 @global_atomic_umax_i64_ret_scalar(ptr addrspace(1) inreg
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[3:4], off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v0, s35
+; SI-NEXT: v_mov_b32_e32 v1, s34
; SI-NEXT: .LBB82_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v8, v4
; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v0, s35
; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[34:35], v[7:8]
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_cndmask_b32_e32 v6, v0, v8, vcc
-; SI-NEXT: v_mov_b32_e32 v0, s34
-; SI-NEXT: v_cndmask_b32_e32 v5, v0, v7, vcc
+; SI-NEXT: v_cndmask_b32_e32 v5, v1, v7, vcc
; SI-NEXT: v_mov_b32_e32 v3, v5
; SI-NEXT: v_mov_b32_e32 v4, v6
; SI-NEXT: v_mov_b32_e32 v5, v7
@@ -6031,24 +6021,24 @@ define amdgpu_gfx i64 @global_atomic_umax_i64_ret_scalar(ptr addrspace(1) inreg
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT: v_mov_b32_e32 v2, s4
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v4, s7
+; VI-NEXT: v_mov_b32_e32 v5, s6
+; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: .LBB82_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_mov_b32_e32 v9, v1
+; VI-NEXT: v_mov_b32_e32 v8, v0
+; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB82_1
@@ -6062,21 +6052,21 @@ define amdgpu_gfx i64 @global_atomic_umax_i64_ret_scalar(ptr addrspace(1) inreg
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
+; GFX9-NEXT: v_mov_b32_e32 v4, s6
; GFX9-NEXT: .LBB82_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v7, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB82_1
@@ -6103,17 +6093,17 @@ define amdgpu_gfx i64 @global_atomic_umax_i64_ret_offset_scalar(ptr addrspace(1)
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[3:4], off, s[4:7], 0 offset:32
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v0, s35
+; SI-NEXT: v_mov_b32_e32 v1, s34
; SI-NEXT: .LBB83_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v8, v4
; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v0, s35
; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[34:35], v[7:8]
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_cndmask_b32_e32 v6, v0, v8, vcc
-; SI-NEXT: v_mov_b32_e32 v0, s34
-; SI-NEXT: v_cndmask_b32_e32 v5, v0, v7, vcc
+; SI-NEXT: v_cndmask_b32_e32 v5, v1, v7, vcc
; SI-NEXT: v_mov_b32_e32 v3, v5
; SI-NEXT: v_mov_b32_e32 v4, v6
; SI-NEXT: v_mov_b32_e32 v5, v7
@@ -6143,32 +6133,30 @@ define amdgpu_gfx i64 @global_atomic_umax_i64_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v2, s34
+; VI-NEXT: v_mov_b32_e32 v3, s35
+; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
+; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v4, s7
+; VI-NEXT: v_mov_b32_e32 v5, s6
; VI-NEXT: .LBB83_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_mov_b32_e32 v9, v1
+; VI-NEXT: v_mov_b32_e32 v8, v0
+; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB83_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_umax_i64_ret_offset_scalar:
@@ -6177,21 +6165,21 @@ define amdgpu_gfx i64 @global_atomic_umax_i64_ret_offset_scalar(ptr addrspace(1)
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
+; GFX9-NEXT: v_mov_b32_e32 v4, s6
; GFX9-NEXT: .LBB83_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v7, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] offset:32 glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB83_1
@@ -6215,30 +6203,30 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr addrspace(1) %out,
; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x8
; SI-NEXT: s_mov_b64 s[0:1], 0
; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: v_mov_b32_e32 v4, s3
+; SI-NEXT: v_mov_b32_e32 v5, s2
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s8
; SI-NEXT: v_mov_b32_e32 v3, s9
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: .LBB84_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: v_mov_b32_e32 v0, s3
; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_mov_b32_e32 v0, s2
-; SI-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
-; SI-NEXT: v_mov_b32_e32 v4, v0
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
+; SI-NEXT: v_mov_b32_e32 v6, v0
; SI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 offset:32 glc
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 offset:32 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; SI-NEXT: v_mov_b32_e32 v2, v4
-; SI-NEXT: v_mov_b32_e32 v3, v5
+; SI-NEXT: v_mov_b32_e32 v2, v6
+; SI-NEXT: v_mov_b32_e32 v3, v7
; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; SI-NEXT: s_cbranch_execnz .LBB84_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6246,28 +6234,28 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr addrspace(1) %out,
;
; VI-LABEL: atomic_umax_i64_addr64_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
+; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x34
; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], 3
-; VI-NEXT: s_add_u32 s0, s0, s4
-; VI-NEXT: s_addc_u32 s1, s1, s5
+; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
+; VI-NEXT: s_add_u32 s0, s0, s6
+; VI-NEXT: s_addc_u32 s1, s1, s7
; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x20
; VI-NEXT: s_add_u32 s0, s0, 32
; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: v_mov_b32_e32 v5, s1
+; VI-NEXT: v_mov_b32_e32 v6, s3
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s6
+; VI-NEXT: v_mov_b32_e32 v7, s2
; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: v_mov_b32_e32 v4, s0
; VI-NEXT: .LBB84_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s3
-; VI-NEXT: v_mov_b32_e32 v6, s2
-; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -6285,25 +6273,25 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr addrspace(1) %out,
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[0:1], s[2:3], 3
; GFX9-NEXT: s_add_u32 s0, s4, s0
; GFX9-NEXT: s_addc_u32 s1, s5, s1
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x20
; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s7
+; GFX9-NEXT: v_mov_b32_e32 v5, s6
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s4
; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB84_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] offset:32 glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -6332,17 +6320,17 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr addrspace(1) %o
; SI-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
; SI-NEXT: s_mov_b64 s[0:1], 0
; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: v_mov_b32_e32 v8, s5
+; SI-NEXT: v_mov_b32_e32 v9, s4
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: s_mov_b32 s10, -1
; SI-NEXT: .LBB85_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: v_mov_b32_e32 v0, s5
; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; SI-NEXT: v_cndmask_b32_e32 v1, v8, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v9, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v7, v3
; SI-NEXT: v_mov_b32_e32 v6, v2
@@ -6370,70 +6358,70 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr addrspace(1) %o
; VI-LABEL: atomic_umax_i64_ret_addr64_offset:
; VI: ; %bb.0: ; %entry
; VI-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24
+; VI-NEXT: s_mov_b64 s[8:9], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; VI-NEXT: s_add_u32 s0, s0, s6
; VI-NEXT: s_addc_u32 s1, s1, s7
-; VI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x20
+; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x20
; VI-NEXT: s_add_u32 s0, s0, 32
; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: s_mov_b64 s[6:7], 0
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v4, s5
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s8
-; VI-NEXT: v_mov_b32_e32 v1, s9
+; VI-NEXT: v_mov_b32_e32 v2, s6
+; VI-NEXT: v_mov_b32_e32 v5, s4
+; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: .LBB85_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s5
-; VI-NEXT: v_mov_b32_e32 v6, s4
-; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_mov_b32_e32 v9, v3
+; VI-NEXT: v_mov_b32_e32 v8, v2
+; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; VI-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; VI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; VI-NEXT: s_andn2_b64 exec, exec, s[8:9]
; VI-NEXT: s_cbranch_execnz .LBB85_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[6:7]
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_or_b64 exec, exec, s[8:9]
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_umax_i64_ret_addr64_offset:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; GFX9-NEXT: s_add_u32 s0, s0, s6
; GFX9-NEXT: s_addc_u32 s1, s1, s7
; GFX9-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x20
; GFX9-NEXT: s_mov_b64 s[6:7], 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s5
+; GFX9-NEXT: v_mov_b32_e32 v3, s4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s8
; GFX9-NEXT: v_mov_b32_e32 v1, s9
; GFX9-NEXT: .LBB85_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s5
-; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[0:1] offset:32 glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[5:8], s[0:1] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX9-NEXT: s_cbranch_execnz .LBB85_1
@@ -6461,17 +6449,17 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr addrspace(1) %out, ptr
; SI-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
; SI-NEXT: s_mov_b64 s[0:1], 0
; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: v_mov_b32_e32 v8, s5
+; SI-NEXT: v_mov_b32_e32 v9, s4
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: s_mov_b32 s10, -1
; SI-NEXT: .LBB86_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: v_mov_b32_e32 v0, s5
; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; SI-NEXT: v_cndmask_b32_e32 v1, v8, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v9, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v7, v3
; SI-NEXT: v_mov_b32_e32 v6, v2
@@ -6501,66 +6489,66 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr addrspace(1) %out, ptr
; VI-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
-; VI-NEXT: s_add_u32 s0, s0, s6
-; VI-NEXT: s_addc_u32 s1, s1, s7
-; VI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
-; VI-NEXT: s_mov_b64 s[6:7], 0
+; VI-NEXT: s_add_u32 s6, s0, s6
+; VI-NEXT: s_addc_u32 s7, s1, s7
+; VI-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0
+; VI-NEXT: v_mov_b32_e32 v0, s6
+; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: v_mov_b32_e32 v4, s5
+; VI-NEXT: v_mov_b32_e32 v5, s4
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s8
-; VI-NEXT: v_mov_b32_e32 v1, s9
+; VI-NEXT: v_mov_b32_e32 v2, s8
+; VI-NEXT: v_mov_b32_e32 v3, s9
+; VI-NEXT: v_mov_b32_e32 v1, s7
; VI-NEXT: .LBB86_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s5
-; VI-NEXT: v_mov_b32_e32 v6, s4
-; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_mov_b32_e32 v9, v3
+; VI-NEXT: v_mov_b32_e32 v8, v2
+; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; VI-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB86_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[6:7]
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_or_b64 exec, exec, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_umax_i64_ret_addr64:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; GFX9-NEXT: s_add_u32 s0, s0, s6
; GFX9-NEXT: s_addc_u32 s1, s1, s7
; GFX9-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
; GFX9-NEXT: s_mov_b64 s[6:7], 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s5
+; GFX9-NEXT: v_mov_b32_e32 v3, s4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s8
; GFX9-NEXT: v_mov_b32_e32 v1, s9
; GFX9-NEXT: .LBB86_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s5
-; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[0:1] glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[5:8], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX9-NEXT: s_cbranch_execnz .LBB86_1
@@ -6972,27 +6960,27 @@ define amdgpu_gfx void @global_atomic_umin_i64_noret_scalar(ptr addrspace(1) inr
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[3:4], off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v5, s35
+; SI-NEXT: v_mov_b32_e32 v6, s34
; SI-NEXT: .LBB91_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: v_mov_b32_e32 v1, s35
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cmp_ge_u64_e32 vcc, s[34:35], v[3:4]
-; SI-NEXT: v_cndmask_b32_e32 v2, v1, v4, vcc
-; SI-NEXT: v_mov_b32_e32 v1, s34
-; SI-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
+; SI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v8, v4
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
+; SI-NEXT: v_mov_b32_e32 v10, v4
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
; SI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[5:8], off, s[4:7], 0 glc
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[7:10], off, s[4:7], 0 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[5:6], v[3:4]
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[3:4]
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; SI-NEXT: v_mov_b32_e32 v3, v5
-; SI-NEXT: v_mov_b32_e32 v4, v6
+; SI-NEXT: v_mov_b32_e32 v3, v7
+; SI-NEXT: v_mov_b32_e32 v4, v8
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
; SI-NEXT: s_cbranch_execnz .LBB91_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7011,17 +6999,17 @@ define amdgpu_gfx void @global_atomic_umin_i64_noret_scalar(ptr addrspace(1) inr
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: v_mov_b32_e32 v4, s4
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v6, s7
+; VI-NEXT: v_mov_b32_e32 v7, s6
+; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: .LBB91_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -7042,14 +7030,14 @@ define amdgpu_gfx void @global_atomic_umin_i64_noret_scalar(ptr addrspace(1) inr
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s7
+; GFX9-NEXT: v_mov_b32_e32 v6, s6
; GFX9-NEXT: .LBB91_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -7083,27 +7071,27 @@ define amdgpu_gfx void @global_atomic_umin_i64_noret_offset_scalar(ptr addrspace
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[3:4], off, s[4:7], 0 offset:32
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v5, s35
+; SI-NEXT: v_mov_b32_e32 v6, s34
; SI-NEXT: .LBB92_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: v_mov_b32_e32 v1, s35
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cmp_ge_u64_e32 vcc, s[34:35], v[3:4]
-; SI-NEXT: v_cndmask_b32_e32 v2, v1, v4, vcc
-; SI-NEXT: v_mov_b32_e32 v1, s34
-; SI-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
+; SI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v8, v4
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
+; SI-NEXT: v_mov_b32_e32 v10, v4
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
; SI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[5:8], off, s[4:7], 0 offset:32 glc
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[7:10], off, s[4:7], 0 offset:32 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[5:6], v[3:4]
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[3:4]
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; SI-NEXT: v_mov_b32_e32 v3, v5
-; SI-NEXT: v_mov_b32_e32 v4, v6
+; SI-NEXT: v_mov_b32_e32 v3, v7
+; SI-NEXT: v_mov_b32_e32 v4, v8
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
; SI-NEXT: s_cbranch_execnz .LBB92_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7121,32 +7109,30 @@ define amdgpu_gfx void @global_atomic_umin_i64_noret_offset_scalar(ptr addrspace
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v4, s34
+; VI-NEXT: v_mov_b32_e32 v5, s35
+; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v6, s7
+; VI-NEXT: v_mov_b32_e32 v7, s6
; VI-NEXT: .LBB92_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB92_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_umin_i64_noret_offset_scalar:
@@ -7155,14 +7141,14 @@ define amdgpu_gfx void @global_atomic_umin_i64_noret_offset_scalar(ptr addrspace
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s7
+; GFX9-NEXT: v_mov_b32_e32 v6, s6
; GFX9-NEXT: .LBB92_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -7197,17 +7183,17 @@ define amdgpu_gfx i64 @global_atomic_umin_i64_ret_scalar(ptr addrspace(1) inreg
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[3:4], off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v0, s35
+; SI-NEXT: v_mov_b32_e32 v1, s34
; SI-NEXT: .LBB93_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v8, v4
; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v0, s35
; SI-NEXT: v_cmp_ge_u64_e32 vcc, s[34:35], v[7:8]
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_cndmask_b32_e32 v6, v0, v8, vcc
-; SI-NEXT: v_mov_b32_e32 v0, s34
-; SI-NEXT: v_cndmask_b32_e32 v5, v0, v7, vcc
+; SI-NEXT: v_cndmask_b32_e32 v5, v1, v7, vcc
; SI-NEXT: v_mov_b32_e32 v3, v5
; SI-NEXT: v_mov_b32_e32 v4, v6
; SI-NEXT: v_mov_b32_e32 v5, v7
@@ -7238,24 +7224,24 @@ define amdgpu_gfx i64 @global_atomic_umin_i64_ret_scalar(ptr addrspace(1) inreg
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT: v_mov_b32_e32 v2, s4
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v4, s7
+; VI-NEXT: v_mov_b32_e32 v5, s6
+; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: .LBB93_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_mov_b32_e32 v9, v1
+; VI-NEXT: v_mov_b32_e32 v8, v0
+; VI-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB93_1
@@ -7269,21 +7255,21 @@ define amdgpu_gfx i64 @global_atomic_umin_i64_ret_scalar(ptr addrspace(1) inreg
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
+; GFX9-NEXT: v_mov_b32_e32 v4, s6
; GFX9-NEXT: .LBB93_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v7, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB93_1
@@ -7310,17 +7296,17 @@ define amdgpu_gfx i64 @global_atomic_umin_i64_ret_offset_scalar(ptr addrspace(1)
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[3:4], off, s[4:7], 0 offset:32
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v0, s35
+; SI-NEXT: v_mov_b32_e32 v1, s34
; SI-NEXT: .LBB94_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v8, v4
; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v0, s35
; SI-NEXT: v_cmp_ge_u64_e32 vcc, s[34:35], v[7:8]
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_cndmask_b32_e32 v6, v0, v8, vcc
-; SI-NEXT: v_mov_b32_e32 v0, s34
-; SI-NEXT: v_cndmask_b32_e32 v5, v0, v7, vcc
+; SI-NEXT: v_cndmask_b32_e32 v5, v1, v7, vcc
; SI-NEXT: v_mov_b32_e32 v3, v5
; SI-NEXT: v_mov_b32_e32 v4, v6
; SI-NEXT: v_mov_b32_e32 v5, v7
@@ -7350,32 +7336,30 @@ define amdgpu_gfx i64 @global_atomic_umin_i64_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v2, s34
+; VI-NEXT: v_mov_b32_e32 v3, s35
+; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
+; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v4, s7
+; VI-NEXT: v_mov_b32_e32 v5, s6
; VI-NEXT: .LBB94_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_mov_b32_e32 v9, v1
+; VI-NEXT: v_mov_b32_e32 v8, v0
+; VI-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB94_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_umin_i64_ret_offset_scalar:
@@ -7384,21 +7368,21 @@ define amdgpu_gfx i64 @global_atomic_umin_i64_ret_offset_scalar(ptr addrspace(1)
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
+; GFX9-NEXT: v_mov_b32_e32 v4, s6
; GFX9-NEXT: .LBB94_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v7, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] offset:32 glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB94_1
@@ -7806,27 +7790,27 @@ define amdgpu_gfx void @global_atomic_min_i64_noret_scalar(ptr addrspace(1) inre
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[3:4], off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v5, s35
+; SI-NEXT: v_mov_b32_e32 v6, s34
; SI-NEXT: .LBB99_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: v_mov_b32_e32 v1, s35
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[34:35], v[3:4]
-; SI-NEXT: v_cndmask_b32_e32 v2, v1, v4, vcc
-; SI-NEXT: v_mov_b32_e32 v1, s34
-; SI-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
+; SI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v8, v4
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
+; SI-NEXT: v_mov_b32_e32 v10, v4
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
; SI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[5:8], off, s[4:7], 0 glc
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[7:10], off, s[4:7], 0 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[5:6], v[3:4]
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[3:4]
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; SI-NEXT: v_mov_b32_e32 v3, v5
-; SI-NEXT: v_mov_b32_e32 v4, v6
+; SI-NEXT: v_mov_b32_e32 v3, v7
+; SI-NEXT: v_mov_b32_e32 v4, v8
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
; SI-NEXT: s_cbranch_execnz .LBB99_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7845,17 +7829,17 @@ define amdgpu_gfx void @global_atomic_min_i64_noret_scalar(ptr addrspace(1) inre
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: v_mov_b32_e32 v4, s4
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v6, s7
+; VI-NEXT: v_mov_b32_e32 v7, s6
+; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: .LBB99_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -7876,14 +7860,14 @@ define amdgpu_gfx void @global_atomic_min_i64_noret_scalar(ptr addrspace(1) inre
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s7
+; GFX9-NEXT: v_mov_b32_e32 v6, s6
; GFX9-NEXT: .LBB99_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -7917,27 +7901,27 @@ define amdgpu_gfx void @global_atomic_min_i64_noret_offset_scalar(ptr addrspace(
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[3:4], off, s[4:7], 0 offset:32
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v5, s35
+; SI-NEXT: v_mov_b32_e32 v6, s34
; SI-NEXT: .LBB100_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: v_mov_b32_e32 v1, s35
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[34:35], v[3:4]
-; SI-NEXT: v_cndmask_b32_e32 v2, v1, v4, vcc
-; SI-NEXT: v_mov_b32_e32 v1, s34
-; SI-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v2, v5, v4, vcc
+; SI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v8, v4
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
+; SI-NEXT: v_mov_b32_e32 v10, v4
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
; SI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[5:8], off, s[4:7], 0 offset:32 glc
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[7:10], off, s[4:7], 0 offset:32 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[5:6], v[3:4]
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[3:4]
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; SI-NEXT: v_mov_b32_e32 v3, v5
-; SI-NEXT: v_mov_b32_e32 v4, v6
+; SI-NEXT: v_mov_b32_e32 v3, v7
+; SI-NEXT: v_mov_b32_e32 v4, v8
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
; SI-NEXT: s_cbranch_execnz .LBB100_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7955,32 +7939,30 @@ define amdgpu_gfx void @global_atomic_min_i64_noret_offset_scalar(ptr addrspace(
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v4, s34
+; VI-NEXT: v_mov_b32_e32 v5, s35
+; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v6, s7
+; VI-NEXT: v_mov_b32_e32 v7, s6
; VI-NEXT: .LBB100_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB100_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_min_i64_noret_offset_scalar:
@@ -7989,14 +7971,14 @@ define amdgpu_gfx void @global_atomic_min_i64_noret_offset_scalar(ptr addrspace(
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s7
+; GFX9-NEXT: v_mov_b32_e32 v6, s6
; GFX9-NEXT: .LBB100_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -8031,17 +8013,17 @@ define amdgpu_gfx i64 @global_atomic_min_i64_ret_scalar(ptr addrspace(1) inreg %
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[3:4], off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v0, s35
+; SI-NEXT: v_mov_b32_e32 v1, s34
; SI-NEXT: .LBB101_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v8, v4
; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v0, s35
; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[34:35], v[7:8]
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_cndmask_b32_e32 v6, v0, v8, vcc
-; SI-NEXT: v_mov_b32_e32 v0, s34
-; SI-NEXT: v_cndmask_b32_e32 v5, v0, v7, vcc
+; SI-NEXT: v_cndmask_b32_e32 v5, v1, v7, vcc
; SI-NEXT: v_mov_b32_e32 v3, v5
; SI-NEXT: v_mov_b32_e32 v4, v6
; SI-NEXT: v_mov_b32_e32 v5, v7
@@ -8072,24 +8054,24 @@ define amdgpu_gfx i64 @global_atomic_min_i64_ret_scalar(ptr addrspace(1) inreg %
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; VI-NEXT: v_mov_b32_e32 v2, s4
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v4, s7
+; VI-NEXT: v_mov_b32_e32 v5, s6
+; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: .LBB101_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_mov_b32_e32 v9, v1
+; VI-NEXT: v_mov_b32_e32 v8, v0
+; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB101_1
@@ -8103,21 +8085,21 @@ define amdgpu_gfx i64 @global_atomic_min_i64_ret_scalar(ptr addrspace(1) inreg %
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
+; GFX9-NEXT: v_mov_b32_e32 v4, s6
; GFX9-NEXT: .LBB101_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v7, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB101_1
@@ -8144,17 +8126,17 @@ define amdgpu_gfx i64 @global_atomic_min_i64_ret_offset_scalar(ptr addrspace(1)
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[3:4], off, s[4:7], 0 offset:32
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v0, s35
+; SI-NEXT: v_mov_b32_e32 v1, s34
; SI-NEXT: .LBB102_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_mov_b32_e32 v8, v4
; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v0, s35
; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[34:35], v[7:8]
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_cndmask_b32_e32 v6, v0, v8, vcc
-; SI-NEXT: v_mov_b32_e32 v0, s34
-; SI-NEXT: v_cndmask_b32_e32 v5, v0, v7, vcc
+; SI-NEXT: v_cndmask_b32_e32 v5, v1, v7, vcc
; SI-NEXT: v_mov_b32_e32 v3, v5
; SI-NEXT: v_mov_b32_e32 v4, v6
; SI-NEXT: v_mov_b32_e32 v5, v7
@@ -8184,32 +8166,30 @@ define amdgpu_gfx i64 @global_atomic_min_i64_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v2, s34
+; VI-NEXT: v_mov_b32_e32 v3, s35
+; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
+; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v4, s7
+; VI-NEXT: v_mov_b32_e32 v5, s6
; VI-NEXT: .LBB102_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_mov_b32_e32 v9, v1
+; VI-NEXT: v_mov_b32_e32 v8, v0
+; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB102_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_min_i64_ret_offset_scalar:
@@ -8218,21 +8198,21 @@ define amdgpu_gfx i64 @global_atomic_min_i64_ret_offset_scalar(ptr addrspace(1)
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
+; GFX9-NEXT: v_mov_b32_e32 v4, s6
; GFX9-NEXT: .LBB102_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v7, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] offset:32 glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB102_1
@@ -8256,30 +8236,30 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr addrspace(1) %out, i
; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x8
; SI-NEXT: s_mov_b64 s[0:1], 0
; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: v_mov_b32_e32 v4, s3
+; SI-NEXT: v_mov_b32_e32 v5, s2
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s8
; SI-NEXT: v_mov_b32_e32 v3, s9
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: .LBB103_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: v_mov_b32_e32 v0, s3
; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_mov_b32_e32 v0, s2
-; SI-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
-; SI-NEXT: v_mov_b32_e32 v4, v0
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
+; SI-NEXT: v_mov_b32_e32 v6, v0
; SI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 offset:32 glc
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 offset:32 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; SI-NEXT: v_mov_b32_e32 v2, v4
-; SI-NEXT: v_mov_b32_e32 v3, v5
+; SI-NEXT: v_mov_b32_e32 v2, v6
+; SI-NEXT: v_mov_b32_e32 v3, v7
; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; SI-NEXT: s_cbranch_execnz .LBB103_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8287,28 +8267,28 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr addrspace(1) %out, i
;
; VI-LABEL: atomic_min_i64_addr64_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34
+; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x34
; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], 3
-; VI-NEXT: s_add_u32 s0, s0, s4
-; VI-NEXT: s_addc_u32 s1, s1, s5
+; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
+; VI-NEXT: s_add_u32 s0, s0, s6
+; VI-NEXT: s_addc_u32 s1, s1, s7
; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x20
; VI-NEXT: s_add_u32 s0, s0, 32
; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: v_mov_b32_e32 v5, s1
+; VI-NEXT: v_mov_b32_e32 v6, s3
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s6
+; VI-NEXT: v_mov_b32_e32 v7, s2
; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: v_mov_b32_e32 v4, s0
; VI-NEXT: .LBB103_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s3
-; VI-NEXT: v_mov_b32_e32 v6, s2
-; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -8326,25 +8306,25 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr addrspace(1) %out, i
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34
; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[0:1], s[2:3], 3
; GFX9-NEXT: s_add_u32 s0, s4, s0
; GFX9-NEXT: s_addc_u32 s1, s5, s1
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x20
; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s7
+; GFX9-NEXT: v_mov_b32_e32 v5, s6
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s4
; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB103_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] offset:32 glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -8373,17 +8353,17 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr addrspace(1) %ou
; SI-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
; SI-NEXT: s_mov_b64 s[0:1], 0
; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: v_mov_b32_e32 v8, s5
+; SI-NEXT: v_mov_b32_e32 v9, s4
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: s_mov_b32 s10, -1
; SI-NEXT: .LBB104_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: v_mov_b32_e32 v0, s5
; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; SI-NEXT: v_cndmask_b32_e32 v1, v8, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v9, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v7, v3
; SI-NEXT: v_mov_b32_e32 v6, v2
@@ -8411,70 +8391,70 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr addrspace(1) %ou
; VI-LABEL: atomic_min_i64_ret_addr64_offset:
; VI: ; %bb.0: ; %entry
; VI-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24
+; VI-NEXT: s_mov_b64 s[8:9], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; VI-NEXT: s_add_u32 s0, s0, s6
; VI-NEXT: s_addc_u32 s1, s1, s7
-; VI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x20
+; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x20
; VI-NEXT: s_add_u32 s0, s0, 32
; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: s_mov_b64 s[6:7], 0
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v4, s5
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s8
-; VI-NEXT: v_mov_b32_e32 v1, s9
+; VI-NEXT: v_mov_b32_e32 v2, s6
+; VI-NEXT: v_mov_b32_e32 v5, s4
+; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: .LBB104_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s5
-; VI-NEXT: v_mov_b32_e32 v6, s4
-; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_mov_b32_e32 v9, v3
+; VI-NEXT: v_mov_b32_e32 v8, v2
+; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; VI-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; VI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; VI-NEXT: s_andn2_b64 exec, exec, s[8:9]
; VI-NEXT: s_cbranch_execnz .LBB104_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[6:7]
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_or_b64 exec, exec, s[8:9]
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_min_i64_ret_addr64_offset:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; GFX9-NEXT: s_add_u32 s0, s0, s6
; GFX9-NEXT: s_addc_u32 s1, s1, s7
; GFX9-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x20
; GFX9-NEXT: s_mov_b64 s[6:7], 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s5
+; GFX9-NEXT: v_mov_b32_e32 v3, s4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s8
; GFX9-NEXT: v_mov_b32_e32 v1, s9
; GFX9-NEXT: .LBB104_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s5
-; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[0:1] offset:32 glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[5:8], s[0:1] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX9-NEXT: s_cbranch_execnz .LBB104_1
@@ -8496,36 +8476,36 @@ define amdgpu_kernel void @atomic_min_i64(ptr addrspace(1) %out, i64 %in) {
; SI: ; %bb.0: ; %entry
; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
-; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: s_mov_b32 s6, s2
-; SI-NEXT: s_mov_b32 s7, s3
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx2 s[10:11], s[0:1], 0x0
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: v_mov_b32_e32 v4, s3
+; SI-NEXT: v_mov_b32_e32 v5, s2
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_mov_b32 s4, s0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v3, s9
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v2, s10
+; SI-NEXT: v_mov_b32_e32 v3, s11
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: .LBB105_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: v_mov_b32_e32 v0, s7
-; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
+; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
-; SI-NEXT: v_mov_b32_e32 v4, v0
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
+; SI-NEXT: v_mov_b32_e32 v6, v0
; SI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[0:3], 0 glc
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
-; SI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; SI-NEXT: v_mov_b32_e32 v2, v4
-; SI-NEXT: v_mov_b32_e32 v3, v5
-; SI-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; SI-NEXT: v_mov_b32_e32 v2, v6
+; SI-NEXT: v_mov_b32_e32 v3, v7
+; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]
; SI-NEXT: s_cbranch_execnz .LBB105_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_endpgm
@@ -8536,18 +8516,18 @@ define amdgpu_kernel void @atomic_min_i64(ptr addrspace(1) %out, i64 %in) {
; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
+; VI-NEXT: v_mov_b32_e32 v5, s1
+; VI-NEXT: v_mov_b32_e32 v6, s3
+; VI-NEXT: v_mov_b32_e32 v7, s2
+; VI-NEXT: v_mov_b32_e32 v4, s0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s6
; VI-NEXT: v_mov_b32_e32 v3, s7
; VI-NEXT: .LBB105_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s3
-; VI-NEXT: v_mov_b32_e32 v6, s2
-; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -8565,21 +8545,21 @@ define amdgpu_kernel void @atomic_min_i64(ptr addrspace(1) %out, i64 %in) {
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX9-NEXT: v_mov_b32_e32 v4, s3
+; GFX9-NEXT: v_mov_b32_e32 v5, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s6
; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: .LBB105_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_mov_b32_e32 v5, s2
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -8606,17 +8586,17 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr addrspace(1) %out, ptr
; SI-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
; SI-NEXT: s_mov_b64 s[0:1], 0
; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: v_mov_b32_e32 v8, s5
+; SI-NEXT: v_mov_b32_e32 v9, s4
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: s_mov_b32 s10, -1
; SI-NEXT: .LBB106_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: v_mov_b32_e32 v0, s5
; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; SI-NEXT: v_cndmask_b32_e32 v1, v8, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v9, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v7, v3
; SI-NEXT: v_mov_b32_e32 v6, v2
@@ -8646,66 +8626,66 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr addrspace(1) %out, ptr
; VI-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
-; VI-NEXT: s_add_u32 s0, s0, s6
-; VI-NEXT: s_addc_u32 s1, s1, s7
-; VI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
-; VI-NEXT: s_mov_b64 s[6:7], 0
+; VI-NEXT: s_add_u32 s6, s0, s6
+; VI-NEXT: s_addc_u32 s7, s1, s7
+; VI-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0
+; VI-NEXT: v_mov_b32_e32 v0, s6
+; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: v_mov_b32_e32 v4, s5
+; VI-NEXT: v_mov_b32_e32 v5, s4
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s8
-; VI-NEXT: v_mov_b32_e32 v1, s9
+; VI-NEXT: v_mov_b32_e32 v2, s8
+; VI-NEXT: v_mov_b32_e32 v3, s9
+; VI-NEXT: v_mov_b32_e32 v1, s7
; VI-NEXT: .LBB106_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s5
-; VI-NEXT: v_mov_b32_e32 v6, s4
-; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_mov_b32_e32 v9, v3
+; VI-NEXT: v_mov_b32_e32 v8, v2
+; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; VI-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB106_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[6:7]
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_or_b64 exec, exec, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_min_i64_ret_addr64:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; GFX9-NEXT: s_add_u32 s0, s0, s6
; GFX9-NEXT: s_addc_u32 s1, s1, s7
; GFX9-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
; GFX9-NEXT: s_mov_b64 s[6:7], 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s5
+; GFX9-NEXT: v_mov_b32_e32 v3, s4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s8
; GFX9-NEXT: v_mov_b32_e32 v1, s9
; GFX9-NEXT: .LBB106_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s5
-; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[0:1] glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[5:8], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX9-NEXT: s_cbranch_execnz .LBB106_1
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll b/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll
index 3641bd4ef865dc4..38e79350c0e181a 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomics-fp.ll
@@ -331,22 +331,21 @@ define amdgpu_kernel void @lds_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3) %
; GFX7-NEXT: s_cbranch_execz .LBB2_4
; GFX7-NEXT: ; %bb.1:
; GFX7-NEXT: s_lshl_b32 s8, s3, 3
-; GFX7-NEXT: v_mov_b32_e32 v1, s8
-; GFX7-NEXT: ds_read_b32 v1, v1
+; GFX7-NEXT: v_mov_b32_e32 v2, s8
+; GFX7-NEXT: ds_read_b32 v1, v2
; GFX7-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s6
-; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v3, s6
+; GFX7-NEXT: v_mul_f32_e32 v3, 0x42280000, v3
; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s8
-; GFX7-NEXT: v_add_f32_e32 v4, v3, v2
+; GFX7-NEXT: v_mov_b32_e32 v4, v1
+; GFX7-NEXT: v_add_f32_e32 v1, v4, v3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v1, v3, v4
+; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB2_2
@@ -364,47 +363,45 @@ define amdgpu_kernel void @lds_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3) %
; GFX7-NEXT: ; %bb.5:
; GFX7-NEXT: s_lshl_b32 s3, s3, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s3
-; GFX7-NEXT: ds_read_b32 v2, v1
-; GFX7-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v1, s6
-; GFX7-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
+; GFX7-NEXT: ds_read_b32 v3, v1
+; GFX7-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
+; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: .LBB2_6: ; %atomicrmw.start2
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_add_f32_e32 v3, v2, v1
-; GFX7-NEXT: v_mov_b32_e32 v4, s3
+; GFX7-NEXT: v_add_f32_e32 v4, v3, v2
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v4, v2, v3
+; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, v3
+; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB2_6
; GFX7-NEXT: .LBB2_7: ; %Flow14
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v1, s2
-; GFX7-NEXT: ds_read_b32 v1, v1
+; GFX7-NEXT: v_mov_b32_e32 v2, s2
+; GFX7-NEXT: ds_read_b32 v1, v2
; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; GFX7-NEXT: v_add_f32_e32 v0, s8, v0
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: s_mov_b64 s[2:3], 0
; GFX7-NEXT: .LBB2_8: ; %atomicrmw.start8
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s2
-; GFX7-NEXT: v_add_f32_e32 v3, v2, v0
+; GFX7-NEXT: v_mov_b32_e32 v3, v1
+; GFX7-NEXT: v_add_f32_e32 v1, v3, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v1, v2, v3
+; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v3, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX7-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX7-NEXT: s_cbranch_execnz .LBB2_8
; GFX7-NEXT: ; %bb.9: ; %atomicrmw.end7
-; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
@@ -427,22 +424,21 @@ define amdgpu_kernel void @lds_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3) %
; GFX8-NEXT: s_cbranch_execz .LBB2_4
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_lshl_b32 s8, s3, 3
-; GFX8-NEXT: v_mov_b32_e32 v1, s8
-; GFX8-NEXT: ds_read_b32 v1, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, s8
+; GFX8-NEXT: ds_read_b32 v1, v2
; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v2, s6
-; GFX8-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v3, s6
+; GFX8-NEXT: v_mul_f32_e32 v3, 0x42280000, v3
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB2_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s8
-; GFX8-NEXT: v_add_f32_e32 v4, v3, v2
+; GFX8-NEXT: v_mov_b32_e32 v4, v1
+; GFX8-NEXT: v_add_f32_e32 v1, v4, v3
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v1, v3, v4
+; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB2_2
@@ -460,47 +456,45 @@ define amdgpu_kernel void @lds_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3) %
; GFX8-NEXT: ; %bb.5:
; GFX8-NEXT: s_lshl_b32 s3, s3, 4
; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: ds_read_b32 v2, v1
-; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s6
-; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
+; GFX8-NEXT: ds_read_b32 v3, v1
+; GFX8-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
+; GFX8-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB2_6: ; %atomicrmw.start2
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_add_f32_e32 v3, v2, v1
-; GFX8-NEXT: v_mov_b32_e32 v4, s3
+; GFX8-NEXT: v_add_f32_e32 v4, v3, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v4, v2, v3
+; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB2_6
; GFX8-NEXT: .LBB2_7: ; %Flow16
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: ds_read_b32 v1, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: ds_read_b32 v1, v2
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX8-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; GFX8-NEXT: v_add_f32_e32 v0, s8, v0
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_mov_b64 s[2:3], 0
; GFX8-NEXT: .LBB2_8: ; %atomicrmw.start8
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_add_f32_e32 v3, v2, v0
+; GFX8-NEXT: v_mov_b32_e32 v3, v1
+; GFX8-NEXT: v_add_f32_e32 v1, v3, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v1, v2, v3
+; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v2, v3, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX8-NEXT: s_cbranch_execnz .LBB2_8
; GFX8-NEXT: ; %bb.9: ; %atomicrmw.end7
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_mov_b32 s2, -1
@@ -698,21 +692,20 @@ define amdgpu_kernel void @lds_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrspa
; GFX7-NEXT: s_cbranch_execz .LBB3_4
; GFX7-NEXT: ; %bb.1:
; GFX7-NEXT: s_lshl_b32 s8, s3, 3
-; GFX7-NEXT: v_mov_b32_e32 v1, s8
-; GFX7-NEXT: ds_read_b32 v1, v1
+; GFX7-NEXT: v_mov_b32_e32 v2, s8
+; GFX7-NEXT: ds_read_b32 v1, v2
; GFX7-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s6
-; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v3, s6
+; GFX7-NEXT: v_mul_f32_e32 v3, 0x42280000, v3
; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s8
-; GFX7-NEXT: v_add_f32_e32 v4, v3, v2
-; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v1, v3, v4
+; GFX7-NEXT: v_mov_b32_e32 v4, v1
+; GFX7-NEXT: v_add_f32_e32 v1, v4, v3
+; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB3_2
@@ -730,45 +723,43 @@ define amdgpu_kernel void @lds_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrspa
; GFX7-NEXT: ; %bb.5:
; GFX7-NEXT: s_lshl_b32 s3, s3, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s3
-; GFX7-NEXT: ds_read_b32 v2, v1
-; GFX7-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v1, s6
-; GFX7-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
+; GFX7-NEXT: ds_read_b32 v3, v1
+; GFX7-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
+; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: .LBB3_6: ; %atomicrmw.start2
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_add_f32_e32 v3, v2, v1
-; GFX7-NEXT: v_mov_b32_e32 v4, s3
-; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v4, v2, v3
+; GFX7-NEXT: v_add_f32_e32 v4, v3, v2
+; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, v3
+; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB3_6
; GFX7-NEXT: .LBB3_7: ; %Flow14
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v1, s2
-; GFX7-NEXT: ds_read_b32 v1, v1
+; GFX7-NEXT: v_mov_b32_e32 v2, s2
+; GFX7-NEXT: ds_read_b32 v1, v2
; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; GFX7-NEXT: v_add_f32_e32 v0, s8, v0
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: s_mov_b64 s[2:3], 0
; GFX7-NEXT: .LBB3_8: ; %atomicrmw.start8
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s2
-; GFX7-NEXT: v_add_f32_e32 v3, v2, v0
-; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v1, v2, v3
+; GFX7-NEXT: v_mov_b32_e32 v3, v1
+; GFX7-NEXT: v_add_f32_e32 v1, v3, v0
+; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v3, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX7-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX7-NEXT: s_cbranch_execnz .LBB3_8
; GFX7-NEXT: ; %bb.9: ; %atomicrmw.end7
-; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
@@ -791,21 +782,20 @@ define amdgpu_kernel void @lds_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrspa
; GFX8-NEXT: s_cbranch_execz .LBB3_4
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_lshl_b32 s8, s3, 3
-; GFX8-NEXT: v_mov_b32_e32 v1, s8
-; GFX8-NEXT: ds_read_b32 v1, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, s8
+; GFX8-NEXT: ds_read_b32 v1, v2
; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v2, s6
-; GFX8-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v3, s6
+; GFX8-NEXT: v_mul_f32_e32 v3, 0x42280000, v3
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s8
-; GFX8-NEXT: v_add_f32_e32 v4, v3, v2
-; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v1, v3, v4
+; GFX8-NEXT: v_mov_b32_e32 v4, v1
+; GFX8-NEXT: v_add_f32_e32 v1, v4, v3
+; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v4
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB3_2
@@ -823,45 +813,43 @@ define amdgpu_kernel void @lds_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrspa
; GFX8-NEXT: ; %bb.5:
; GFX8-NEXT: s_lshl_b32 s3, s3, 4
; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: ds_read_b32 v2, v1
-; GFX8-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s6
-; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
+; GFX8-NEXT: ds_read_b32 v3, v1
+; GFX8-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
+; GFX8-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB3_6: ; %atomicrmw.start2
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_add_f32_e32 v3, v2, v1
-; GFX8-NEXT: v_mov_b32_e32 v4, s3
-; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v4, v2, v3
+; GFX8-NEXT: v_add_f32_e32 v4, v3, v2
+; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB3_6
; GFX8-NEXT: .LBB3_7: ; %Flow16
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: ds_read_b32 v1, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: ds_read_b32 v1, v2
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX8-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; GFX8-NEXT: v_add_f32_e32 v0, s8, v0
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_mov_b64 s[2:3], 0
; GFX8-NEXT: .LBB3_8: ; %atomicrmw.start8
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_add_f32_e32 v3, v2, v0
-; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v1, v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v1
+; GFX8-NEXT: v_add_f32_e32 v1, v3, v0
+; GFX8-NEXT: ds_cmpst_rtn_b32 v1, v2, v3, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX8-NEXT: s_cbranch_execnz .LBB3_8
; GFX8-NEXT: ; %bb.9: ; %atomicrmw.end7
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9
; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_mov_b32 s2, -1
diff --git a/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll b/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
index 0e6c1aecb6774a3..615039500962da9 100644
--- a/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
+++ b/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
@@ -69,21 +69,21 @@ define void @issue63986(i64 %0, i64 %idxprom) {
; CHECK-NEXT: v_lshlrev_b64 v[2:3], 6, v[2:3]
; CHECK-NEXT: s_cbranch_execnz .LBB0_9
; CHECK-NEXT: .LBB0_6: ; %loop-memcpy-residual.preheader
+; CHECK-NEXT: v_mov_b32_e32 v7, s5
; CHECK-NEXT: v_or_b32_e32 v2, 32, v4
; CHECK-NEXT: v_mov_b32_e32 v3, v5
; CHECK-NEXT: s_mov_b64 s[6:7], 0
+; CHECK-NEXT: v_mov_b32_e32 v6, s4
; CHECK-NEXT: .LBB0_7: ; %loop-memcpy-residual
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: s_add_u32 s8, 32, s6
-; CHECK-NEXT: s_addc_u32 s9, 0, s7
-; CHECK-NEXT: v_mov_b32_e32 v6, s8
-; CHECK-NEXT: v_mov_b32_e32 v7, s9
-; CHECK-NEXT: flat_load_ubyte v10, v[6:7]
+; CHECK-NEXT: s_add_u32 s4, 32, s6
+; CHECK-NEXT: s_addc_u32 s5, 0, s7
+; CHECK-NEXT: v_mov_b32_e32 v9, s5
+; CHECK-NEXT: v_mov_b32_e32 v8, s4
+; CHECK-NEXT: flat_load_ubyte v10, v[8:9]
; CHECK-NEXT: v_mov_b32_e32 v9, s7
-; CHECK-NEXT: v_mov_b32_e32 v7, s5
; CHECK-NEXT: v_add_co_u32_e32 v8, vcc, s6, v2
; CHECK-NEXT: s_add_u32 s6, s6, 1
-; CHECK-NEXT: v_mov_b32_e32 v6, s4
; CHECK-NEXT: v_addc_co_u32_e32 v9, vcc, v3, v9, vcc
; CHECK-NEXT: s_addc_u32 s7, s7, 0
; CHECK-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[6:7]
@@ -236,13 +236,13 @@ define void @issue63986_reduced_expanded(i64 %idxprom) {
; CHECK-NEXT: v_lshlrev_b64 v[0:1], 1, v[0:1]
; CHECK-NEXT: s_cbranch_execnz .LBB1_8
; CHECK-NEXT: .LBB1_5: ; %loop-memcpy-residual.preheader
-; CHECK-NEXT: s_mov_b64 s[6:7], 0
-; CHECK-NEXT: .LBB1_6: ; %loop-memcpy-residual
-; CHECK-NEXT: s_add_u32 s6, s6, 1
; CHECK-NEXT: v_mov_b32_e32 v0, s4
+; CHECK-NEXT: s_mov_b64 s[6:7], 0
; CHECK-NEXT: v_mov_b32_e32 v1, s5
-; CHECK-NEXT: s_addc_u32 s7, s7, 0
-; CHECK-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
+; CHECK-NEXT: .LBB1_6: ; %loop-memcpy-residual
+; CHECK-NEXT: s_add_u32 s4, s6, 1
+; CHECK-NEXT: s_addc_u32 s5, s7, 0
+; CHECK-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]
; CHECK-NEXT: s_mov_b64 s[6:7], 1
; CHECK-NEXT: s_cbranch_vccnz .LBB1_6
; CHECK-NEXT: ; %bb.7: ; %Flow
diff --git a/llvm/test/CodeGen/AMDGPU/optimize-negated-cond.ll b/llvm/test/CodeGen/AMDGPU/optimize-negated-cond.ll
index f284df4d8a70b1b..19ab6d0854a1dad 100644
--- a/llvm/test/CodeGen/AMDGPU/optimize-negated-cond.ll
+++ b/llvm/test/CodeGen/AMDGPU/optimize-negated-cond.ll
@@ -9,7 +9,8 @@ define amdgpu_kernel void @negated_cond(ptr addrspace(1) %arg1) {
; GCN-NEXT: s_mov_b32 s10, -1
; GCN-NEXT: s_mov_b32 s6, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GCN-NEXT: s_mov_b32 s9, s5
+; GCN-NEXT: s_mov_b32 s8, s4
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_branch .LBB0_2
; GCN-NEXT: .LBB0_1: ; %loop.exit.guard
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
index 3cb03099da93d51..6f61af0418636a9 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
@@ -496,7 +496,8 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1) %buffer) {
; GFX900-NEXT: v_mov_b32_e32 v2, s35
; GFX900-NEXT: v_add_co_u32_e32 v1, vcc, s34, v1
; GFX900-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
-; GFX900-NEXT: v_add_co_u32_e32 v1, vcc, 0x5000, v1
+; GFX900-NEXT: s_movk_i32 s0, 0x5000
+; GFX900-NEXT: v_add_co_u32_e32 v1, vcc, s0, v1
; GFX900-NEXT: v_mov_b32_e32 v3, 0
; GFX900-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
; GFX900-NEXT: s_movk_i32 s2, 0x7f
@@ -723,7 +724,8 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1) %buffer) {
; GFX90A-NEXT: v_mov_b32_e32 v2, s35
; GFX90A-NEXT: v_add_co_u32_e32 v1, vcc, s34, v1
; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v2, vcc
-; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, 0x5000, v1
+; GFX90A-NEXT: s_movk_i32 s0, 0x5000
+; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, s0, v1
; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX90A-NEXT: s_movk_i32 s2, 0x7f
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], 0, 0
diff --git a/llvm/test/CodeGen/AMDGPU/remat-fp64-constants.ll b/llvm/test/CodeGen/AMDGPU/remat-fp64-constants.ll
index 26cde1e6cd59d20..a81a934ef9122d6 100644
--- a/llvm/test/CodeGen/AMDGPU/remat-fp64-constants.ll
+++ b/llvm/test/CodeGen/AMDGPU/remat-fp64-constants.ll
@@ -4,7 +4,7 @@
; GCN-LABEL: {{^}}test_remat_sgpr:
; GCN-NOT: v_writelane_b32
; GCN: {{^}}[[LOOP:.LBB[0-9_]+]]:
-; GCN-COUNT-6: s_mov_b32 s{{[0-9]+}}, 0x
+; GCN-COUNT-4: s_mov_b32 s{{[0-9]+}}, 0x
; GCN-NOT: v_writelane_b32
; GCN: s_cbranch_{{[^ ]+}} [[LOOP]]
; GCN: .sgpr_spill_count: 0
diff --git a/llvm/test/CodeGen/AMDGPU/sdiv64.ll b/llvm/test/CodeGen/AMDGPU/sdiv64.ll
index 9cb6842ae0a1827..adc4c85aa60dc07 100644
--- a/llvm/test/CodeGen/AMDGPU/sdiv64.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdiv64.ll
@@ -1720,8 +1720,8 @@ define i64 @v_test_sdiv_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_addc_u32_e64 v8, s[4:5], 0, -1, vcc
; GCN-IR-NEXT: s_mov_b64 s[10:11], 0
; GCN-IR-NEXT: v_mov_b32_e32 v12, 0
-; GCN-IR-NEXT: v_mov_b32_e32 v6, 0
; GCN-IR-NEXT: s_movk_i32 s12, 0x7fff
+; GCN-IR-NEXT: v_mov_b32_e32 v6, 0
; GCN-IR-NEXT: .LBB13_3: ; %udiv-do-while
; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN-IR-NEXT: v_lshl_b64 v[9:10], v[9:10], 1
diff --git a/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll b/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
index 680c27b69770903..11c71a71f266f58 100644
--- a/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdwa-peephole.ll
@@ -2103,19 +2103,19 @@ define void @crash_lshlrevb16_not_reg_op() {
; NOSDWA: ; %bb.0: ; %bb0
; NOSDWA-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; NOSDWA-NEXT: s_mov_b64 s[4:5], 0
+; NOSDWA-NEXT: v_mov_b32_e32 v0, 0xff
+; NOSDWA-NEXT: v_and_b32_e32 v0, s4, v0
+; NOSDWA-NEXT: v_lshlrev_b16_e64 v1, 8, 1
+; NOSDWA-NEXT: v_or_b32_e32 v0, v0, v1
; NOSDWA-NEXT: s_and_b64 vcc, exec, -1
-; NOSDWA-NEXT: v_lshlrev_b16_e64 v3, 8, 1
; NOSDWA-NEXT: .LBB22_1: ; %bb1
; NOSDWA-NEXT: ; =>This Inner Loop Header: Depth=1
-; NOSDWA-NEXT: v_mov_b32_e32 v0, s4
-; NOSDWA-NEXT: v_mov_b32_e32 v2, 0xff
; NOSDWA-NEXT: s_lshl_b32 s6, s4, 3
-; NOSDWA-NEXT: v_mov_b32_e32 v1, s5
+; NOSDWA-NEXT: v_mov_b32_e32 v1, s4
+; NOSDWA-NEXT: v_mov_b32_e32 v2, s5
; NOSDWA-NEXT: s_mov_b64 s[4:5], 1
-; NOSDWA-NEXT: v_and_b32_e32 v2, s4, v2
-; NOSDWA-NEXT: v_or_b32_e32 v2, v2, v3
-; NOSDWA-NEXT: v_lshrrev_b16_e32 v2, s6, v2
-; NOSDWA-NEXT: flat_store_byte v[0:1], v2
+; NOSDWA-NEXT: v_lshrrev_b16_e32 v3, s6, v0
+; NOSDWA-NEXT: flat_store_byte v[1:2], v3
; NOSDWA-NEXT: s_mov_b64 vcc, vcc
; NOSDWA-NEXT: s_cbranch_vccnz .LBB22_1
; NOSDWA-NEXT: ; %bb.2: ; %DummyReturnBlock
@@ -2126,17 +2126,17 @@ define void @crash_lshlrevb16_not_reg_op() {
; GFX89: ; %bb.0: ; %bb0
; GFX89-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX89-NEXT: s_mov_b64 s[4:5], 0
-; GFX89-NEXT: s_and_b64 vcc, exec, -1
; GFX89-NEXT: v_lshlrev_b16_e64 v0, 8, 1
+; GFX89-NEXT: v_mov_b32_e32 v1, s4
+; GFX89-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX89-NEXT: s_and_b64 vcc, exec, -1
; GFX89-NEXT: .LBB22_1: ; %bb1
; GFX89-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX89-NEXT: v_mov_b32_e32 v3, s4
; GFX89-NEXT: s_lshl_b32 s6, s4, 3
; GFX89-NEXT: v_mov_b32_e32 v1, s4
-; GFX89-NEXT: v_or_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX89-NEXT: v_mov_b32_e32 v2, s5
; GFX89-NEXT: s_mov_b64 s[4:5], 1
-; GFX89-NEXT: v_lshrrev_b16_e32 v3, s6, v3
+; GFX89-NEXT: v_lshrrev_b16_e32 v3, s6, v0
; GFX89-NEXT: flat_store_byte v[1:2], v3
; GFX89-NEXT: s_mov_b64 vcc, vcc
; GFX89-NEXT: s_cbranch_vccnz .LBB22_1
@@ -2149,8 +2149,8 @@ define void @crash_lshlrevb16_not_reg_op() {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: v_lshlrev_b16_e64 v0, 8, 1
-; GFX9-NEXT: s_and_b64 vcc, exec, -1
; GFX9-NEXT: v_or_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_and_b64 vcc, exec, -1
; GFX9-NEXT: .LBB22_1: ; %bb1
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_lshl_b32 s6, s4, 3
diff --git a/llvm/test/CodeGen/AMDGPU/srem64.ll b/llvm/test/CodeGen/AMDGPU/srem64.ll
index 520ec6e24ae3bfe..bfb409438757c73 100644
--- a/llvm/test/CodeGen/AMDGPU/srem64.ll
+++ b/llvm/test/CodeGen/AMDGPU/srem64.ll
@@ -1912,8 +1912,8 @@ define i64 @v_test_srem_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_addc_u32_e64 v9, s[4:5], 0, -1, vcc
; GCN-IR-NEXT: s_mov_b64 s[10:11], 0
; GCN-IR-NEXT: v_mov_b32_e32 v13, 0
-; GCN-IR-NEXT: v_mov_b32_e32 v7, 0
; GCN-IR-NEXT: s_movk_i32 s12, 0x7fff
+; GCN-IR-NEXT: v_mov_b32_e32 v7, 0
; GCN-IR-NEXT: .LBB13_3: ; %udiv-do-while
; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN-IR-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
diff --git a/llvm/test/CodeGen/AMDGPU/udiv64.ll b/llvm/test/CodeGen/AMDGPU/udiv64.ll
index e809292aad1d38b..6102d652402a210 100644
--- a/llvm/test/CodeGen/AMDGPU/udiv64.ll
+++ b/llvm/test/CodeGen/AMDGPU/udiv64.ll
@@ -1281,8 +1281,8 @@ define i64 @v_test_udiv_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_addc_u32_e64 v1, s[4:5], 0, -1, vcc
; GCN-IR-NEXT: s_mov_b64 s[10:11], 0
; GCN-IR-NEXT: v_mov_b32_e32 v10, 0
-; GCN-IR-NEXT: v_mov_b32_e32 v5, 0
; GCN-IR-NEXT: s_movk_i32 s12, 0x7fff
+; GCN-IR-NEXT: v_mov_b32_e32 v5, 0
; GCN-IR-NEXT: .LBB10_3: ; %udiv-do-while
; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN-IR-NEXT: v_lshl_b64 v[7:8], v[7:8], 1
diff --git a/llvm/test/CodeGen/AMDGPU/urem64.ll b/llvm/test/CodeGen/AMDGPU/urem64.ll
index 9c316612528c208..7e8661b3ff32dd1 100644
--- a/llvm/test/CodeGen/AMDGPU/urem64.ll
+++ b/llvm/test/CodeGen/AMDGPU/urem64.ll
@@ -1280,8 +1280,8 @@ define i64 @v_test_urem_pow2_k_den_i64(i64 %x) {
; GCN-IR-NEXT: v_addc_u32_e64 v7, s[4:5], 0, -1, vcc
; GCN-IR-NEXT: s_mov_b64 s[10:11], 0
; GCN-IR-NEXT: v_mov_b32_e32 v11, 0
-; GCN-IR-NEXT: v_mov_b32_e32 v5, 0
; GCN-IR-NEXT: s_movk_i32 s12, 0x7fff
+; GCN-IR-NEXT: v_mov_b32_e32 v5, 0
; GCN-IR-NEXT: .LBB9_3: ; %udiv-do-while
; GCN-IR-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN-IR-NEXT: v_lshl_b64 v[8:9], v[8:9], 1
diff --git a/llvm/test/CodeGen/X86/2007-01-13-StackPtrIndex.ll b/llvm/test/CodeGen/X86/2007-01-13-StackPtrIndex.ll
index ca281043eddaaee..46dddd8fcd851a1 100644
--- a/llvm/test/CodeGen/X86/2007-01-13-StackPtrIndex.ll
+++ b/llvm/test/CodeGen/X86/2007-01-13-StackPtrIndex.ll
@@ -48,14 +48,18 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
; CHECK-NEXT: js .LBB0_10
; CHECK-NEXT: # %bb.2: # %b158
; CHECK-NEXT: movslq (%r9), %rsi
-; CHECK-NEXT: xorl %edi, %edi
+; CHECK-NEXT: movl %esi, %edi
+; CHECK-NEXT: orl %eax, %edi
+; CHECK-NEXT: movl %edi, %r8d
+; CHECK-NEXT: shrl $31, %r8d
+; CHECK-NEXT: xorl %r9d, %r9d
; CHECK-NEXT: xorps %xmm0, %xmm0
-; CHECK-NEXT: movb $1, %r8b
+; CHECK-NEXT: movb $1, %r10b
; CHECK-NEXT: jmp .LBB0_3
; CHECK-NEXT: .p2align 4, 0x90
; CHECK-NEXT: .LBB0_9: # %b1606
; CHECK-NEXT: # in Loop: Header=BB0_3 Depth=1
-; CHECK-NEXT: testb %dil, %dil
+; CHECK-NEXT: testb %r9b, %r9b
; CHECK-NEXT: je .LBB0_10
; CHECK-NEXT: .LBB0_3: # %a29b173
; CHECK-NEXT: # =>This Loop Header: Depth=1
@@ -83,47 +87,42 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
; CHECK-NEXT: .LBB0_36: # %a30b
; CHECK-NEXT: # Parent Loop BB0_3 Depth=1
; CHECK-NEXT: # => This Inner Loop Header: Depth=2
-; CHECK-NEXT: testb %dil, %dil
+; CHECK-NEXT: testb %r9b, %r9b
; CHECK-NEXT: je .LBB0_36
; CHECK-NEXT: .LBB0_18: # %b188
; CHECK-NEXT: # in Loop: Header=BB0_3 Depth=1
-; CHECK-NEXT: testb %r8b, %r8b
+; CHECK-NEXT: testb %r10b, %r10b
; CHECK-NEXT: jne .LBB0_4
; CHECK-NEXT: .p2align 4, 0x90
; CHECK-NEXT: .LBB0_19: # %a30b294
; CHECK-NEXT: # Parent Loop BB0_3 Depth=1
; CHECK-NEXT: # => This Inner Loop Header: Depth=2
-; CHECK-NEXT: testb %dil, %dil
+; CHECK-NEXT: testb %r9b, %r9b
; CHECK-NEXT: je .LBB0_19
; CHECK-NEXT: .LBB0_4: # %a33b
; CHECK-NEXT: # in Loop: Header=BB0_3 Depth=1
-; CHECK-NEXT: movl %esi, %r9d
-; CHECK-NEXT: orl %eax, %r9d
+; CHECK-NEXT: testl %edi, %edi
; CHECK-NEXT: jns .LBB0_20
; CHECK-NEXT: .LBB0_5: # %a50b
; CHECK-NEXT: # in Loop: Header=BB0_3 Depth=1
-; CHECK-NEXT: movl %eax, %r10d
-; CHECK-NEXT: orl %esi, %r10d
+; CHECK-NEXT: testl %edi, %edi
; CHECK-NEXT: jns .LBB0_25
; CHECK-NEXT: .LBB0_6: # %a57b
; CHECK-NEXT: # in Loop: Header=BB0_3 Depth=1
-; CHECK-NEXT: shrl $31, %r9d
-; CHECK-NEXT: testb %r9b, %r9b
+; CHECK-NEXT: testb %r8b, %r8b
; CHECK-NEXT: je .LBB0_29
; CHECK-NEXT: .LBB0_7: # %a66b
; CHECK-NEXT: # in Loop: Header=BB0_3 Depth=1
-; CHECK-NEXT: shrl $31, %r10d
-; CHECK-NEXT: testb %r10b, %r10b
+; CHECK-NEXT: testb %r8b, %r8b
; CHECK-NEXT: jne .LBB0_8
; CHECK-NEXT: .p2align 4, 0x90
; CHECK-NEXT: .LBB0_33: # %a74b
; CHECK-NEXT: # Parent Loop BB0_3 Depth=1
; CHECK-NEXT: # => This Inner Loop Header: Depth=2
-; CHECK-NEXT: testb %dil, %dil
+; CHECK-NEXT: testb %r9b, %r9b
; CHECK-NEXT: jne .LBB0_33
; CHECK-NEXT: # %bb.34: # %b1582
; CHECK-NEXT: # in Loop: Header=BB0_33 Depth=2
-; CHECK-NEXT: testb %dil, %dil
; CHECK-NEXT: jne .LBB0_33
; CHECK-NEXT: .LBB0_8: # %a93b
; CHECK-NEXT: # in Loop: Header=BB0_3 Depth=1
@@ -140,13 +139,13 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
; CHECK-NEXT: addss %xmm0, %xmm2
; CHECK-NEXT: addss %xmm1, %xmm2
; CHECK-NEXT: movss %xmm2, i6000(%rip)
-; CHECK-NEXT: testb %dil, %dil
+; CHECK-NEXT: testb %r9b, %r9b
; CHECK-NEXT: jne .LBB0_35
; CHECK-NEXT: jmp .LBB0_9
; CHECK-NEXT: .p2align 4, 0x90
; CHECK-NEXT: .LBB0_21: # %b377
; CHECK-NEXT: # in Loop: Header=BB0_20 Depth=2
-; CHECK-NEXT: testb %dil, %dil
+; CHECK-NEXT: testb %r9b, %r9b
; CHECK-NEXT: je .LBB0_22
; CHECK-NEXT: .LBB0_20: # %b341
; CHECK-NEXT: # Parent Loop BB0_3 Depth=1
@@ -159,7 +158,7 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
; CHECK-NEXT: # Parent Loop BB0_3 Depth=1
; CHECK-NEXT: # Parent Loop BB0_20 Depth=2
; CHECK-NEXT: # => This Inner Loop Header: Depth=3
-; CHECK-NEXT: testb %dil, %dil
+; CHECK-NEXT: testb %r9b, %r9b
; CHECK-NEXT: je .LBB0_37
; CHECK-NEXT: jmp .LBB0_21
; CHECK-NEXT: .p2align 4, 0x90
@@ -179,11 +178,11 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
; CHECK-NEXT: # Parent Loop BB0_3 Depth=1
; CHECK-NEXT: # Parent Loop BB0_25 Depth=2
; CHECK-NEXT: # => This Inner Loop Header: Depth=3
-; CHECK-NEXT: testb %dil, %dil
+; CHECK-NEXT: testb %r9b, %r9b
; CHECK-NEXT: je .LBB0_38
; CHECK-NEXT: .LBB0_26: # %b879
; CHECK-NEXT: # in Loop: Header=BB0_25 Depth=2
-; CHECK-NEXT: testb %r8b, %r8b
+; CHECK-NEXT: testb %r10b, %r10b
; CHECK-NEXT: jne .LBB0_27
; CHECK-NEXT: .p2align 4, 0x90
; CHECK-NEXT: .LBB0_28: # %a53b1019
@@ -227,7 +226,7 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
; CHECK-NEXT: .p2align 4, 0x90
; CHECK-NEXT: .LBB0_24: # %b712
; CHECK-NEXT: # in Loop: Header=BB0_22 Depth=2
-; CHECK-NEXT: testb %dil, %dil
+; CHECK-NEXT: testb %r9b, %r9b
; CHECK-NEXT: je .LBB0_5
; CHECK-NEXT: .LBB0_22: # %b535
; CHECK-NEXT: # Parent Loop BB0_3 Depth=1
@@ -240,7 +239,7 @@ define dso_local void @foo(ptr %a0, ptr %a1, ptr %a2, ptr %a3, ptr %a4, ptr %a5)
; CHECK-NEXT: # Parent Loop BB0_3 Depth=1
; CHECK-NEXT: # Parent Loop BB0_22 Depth=2
; CHECK-NEXT: # => This Inner Loop Header: Depth=3
-; CHECK-NEXT: testb %dil, %dil
+; CHECK-NEXT: testb %r9b, %r9b
; CHECK-NEXT: je .LBB0_23
; CHECK-NEXT: jmp .LBB0_24
; CHECK-NEXT: .LBB0_10: # %a109b
diff --git a/llvm/test/CodeGen/X86/2009-02-26-MachineLICMBug.ll b/llvm/test/CodeGen/X86/2009-02-26-MachineLICMBug.ll
index f811b2f09ba856e..7807d49269e64cc 100644
--- a/llvm/test/CodeGen/X86/2009-02-26-MachineLICMBug.ll
+++ b/llvm/test/CodeGen/X86/2009-02-26-MachineLICMBug.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; REQUIRES: asserts
-; RUN: llc < %s -mattr=+sse3,+sse4.1 -mcpu=penryn -stats 2>&1 | grep "5 machinelicm"
+; RUN: llc < %s -mattr=+sse3,+sse4.1 -mcpu=penryn -stats 2>&1 | grep "6 machinelicm"
; RUN: llc < %s -mattr=+sse3,+sse4.1 -mcpu=penryn | FileCheck %s
; rdar://6627786
; rdar://7792037
diff --git a/llvm/test/CodeGen/X86/avx512-i1test.ll b/llvm/test/CodeGen/X86/avx512-i1test.ll
index 2c91a9bfb0b9200..bf31b2da56b62eb 100644
--- a/llvm/test/CodeGen/X86/avx512-i1test.ll
+++ b/llvm/test/CodeGen/X86/avx512-i1test.ll
@@ -11,7 +11,7 @@ define void @func() {
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: testb %al, %al
; CHECK-NEXT: je .LBB0_1
-; CHECK-NEXT: # %bb.4: # %L_30
+; CHECK-NEXT: # %bb.3: # %L_30
; CHECK-NEXT: retq
; CHECK-NEXT: .LBB0_1: # %bb56
; CHECK-NEXT: xorl %eax, %eax
@@ -19,10 +19,6 @@ define void @func() {
; CHECK-NEXT: .LBB0_2: # %bb33
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-NEXT: testb %al, %al
-; CHECK-NEXT: jne .LBB0_2
-; CHECK-NEXT: # %bb.3: # %bb35
-; CHECK-NEXT: # in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: testb %al, %al
; CHECK-NEXT: jmp .LBB0_2
bb1:
br i1 undef, label %L_10, label %L_10
diff --git a/llvm/test/CodeGen/X86/pr38795.ll b/llvm/test/CodeGen/X86/pr38795.ll
index 5695ab5e288b5d8..03629a353d84dce 100644
--- a/llvm/test/CodeGen/X86/pr38795.ll
+++ b/llvm/test/CodeGen/X86/pr38795.ll
@@ -118,7 +118,6 @@ define dso_local void @fn() {
; CHECK-NEXT: jne .LBB0_22
; CHECK-NEXT: # %bb.23: # %for.cond47
; CHECK-NEXT: # in Loop: Header=BB0_22 Depth=2
-; CHECK-NEXT: testb %bl, %bl
; CHECK-NEXT: jne .LBB0_22
; CHECK-NEXT: .LBB0_9: # %ae
; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1
diff --git a/llvm/test/CodeGen/X86/pr63108.ll b/llvm/test/CodeGen/X86/pr63108.ll
index 67785ce532966ba..6b0543315d8670f 100644
--- a/llvm/test/CodeGen/X86/pr63108.ll
+++ b/llvm/test/CodeGen/X86/pr63108.ll
@@ -15,17 +15,17 @@ define i32 @PR63108() {
; SSE-NEXT: jmp .LBB0_5
; SSE-NEXT: .LBB0_2: # %vector.body.preheader
; SSE-NEXT: pxor %xmm0, %xmm0
-; SSE-NEXT: movdqa {{.*#+}} xmm2 = [57339,0,0,0]
+; SSE-NEXT: movdqa {{.*#+}} xmm1 = [57339,0,0,0]
; SSE-NEXT: xorl %eax, %eax
; SSE-NEXT: .p2align 4, 0x90
; SSE-NEXT: .LBB0_3: # %vector.body
; SSE-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE-NEXT: movdqa %xmm2, %xmm1
+; SSE-NEXT: movdqa %xmm1, %xmm2
; SSE-NEXT: testb %al, %al
-; SSE-NEXT: pxor %xmm2, %xmm2
+; SSE-NEXT: pxor %xmm1, %xmm1
; SSE-NEXT: jne .LBB0_3
; SSE-NEXT: # %bb.4: # %middle.block
-; SSE-NEXT: pxor %xmm1, %xmm0
+; SSE-NEXT: pxor %xmm2, %xmm0
; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; SSE-NEXT: pxor %xmm0, %xmm1
; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
diff --git a/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll b/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll
index 9294ae48a76956e..fd29d09d91960a0 100644
--- a/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-by-select-loop.ll
@@ -152,87 +152,89 @@ define void @vector_variable_shift_left_loop(ptr nocapture %arr, ptr nocapture r
; AVX1-NEXT: .LBB0_3: # %vector.ph
; AVX1-NEXT: movl %r9d, %edx
; AVX1-NEXT: andl $-32, %edx
-; AVX1-NEXT: vmovd %eax, %xmm0
-; AVX1-NEXT: vmovd %r8d, %xmm1
+; AVX1-NEXT: vmovd %eax, %xmm7
+; AVX1-NEXT: vmovd %r8d, %xmm8
; AVX1-NEXT: xorl %ecx, %ecx
-; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero
-; AVX1-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero
-; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero
-; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm6 = xmm1[0],zero,xmm1[1],zero
-; AVX1-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm7 = xmm0[0],zero,xmm0[1],zero
-; AVX1-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm8 = xmm1[0],zero,xmm1[1],zero
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm7[0],zero,xmm7[1],zero
+; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm8[0],zero,xmm8[1],zero
+; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm7[0],zero,xmm7[1],zero
+; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm8[0],zero,xmm8[1],zero
+; AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm7[0],zero,xmm7[1],zero
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm6 = xmm8[0],zero,xmm8[1],zero
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm7 = xmm7[0],zero,xmm7[1],zero
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm8 = xmm8[0],zero,xmm8[1],zero
; AVX1-NEXT: .p2align 4, 0x90
; AVX1-NEXT: .LBB0_4: # %vector.body
; AVX1-NEXT: # =>This Inner Loop Header: Depth=1
; AVX1-NEXT: vmovq {{.*#+}} xmm9 = mem[0],zero
; AVX1-NEXT: vmovq {{.*#+}} xmm10 = mem[0],zero
-; AVX1-NEXT: vmovq {{.*#+}} xmm13 = mem[0],zero
-; AVX1-NEXT: vmovq {{.*#+}} xmm14 = mem[0],zero
-; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT: vpcmpeqb %xmm3, %xmm9, %xmm9
+; AVX1-NEXT: vmovq {{.*#+}} xmm11 = mem[0],zero
+; AVX1-NEXT: vmovq {{.*#+}} xmm12 = mem[0],zero
+; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpcmpeqb %xmm2, %xmm9, %xmm9
+; AVX1-NEXT: vpmovsxbd %xmm9, %xmm14
+; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[1,1,1,1]
; AVX1-NEXT: vpmovsxbd %xmm9, %xmm15
+; AVX1-NEXT: vpcmpeqb %xmm2, %xmm10, %xmm9
+; AVX1-NEXT: vpmovsxbd %xmm9, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[1,1,1,1]
-; AVX1-NEXT: vpmovsxbd %xmm9, %xmm12
-; AVX1-NEXT: vpcmpeqb %xmm3, %xmm10, %xmm9
-; AVX1-NEXT: vpmovsxbd %xmm9, %xmm11
+; AVX1-NEXT: vpmovsxbd %xmm9, %xmm1
+; AVX1-NEXT: vpcmpeqb %xmm2, %xmm11, %xmm9
+; AVX1-NEXT: vpmovsxbd %xmm9, %xmm13
; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[1,1,1,1]
+; AVX1-NEXT: vpmovsxbd %xmm9, %xmm11
+; AVX1-NEXT: vpcmpeqb %xmm2, %xmm12, %xmm9
; AVX1-NEXT: vpmovsxbd %xmm9, %xmm10
-; AVX1-NEXT: vpcmpeqb %xmm3, %xmm13, %xmm13
-; AVX1-NEXT: vpmovzxdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
-; AVX1-NEXT: # xmm0 = mem[0],zero,mem[1],zero
-; AVX1-NEXT: vmovdqu (%rdi,%rcx,4), %xmm9
-; AVX1-NEXT: vpslld %xmm0, %xmm9, %xmm1
-; AVX1-NEXT: vpmovzxdq {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
-; AVX1-NEXT: # xmm2 = mem[0],zero,mem[1],zero
-; AVX1-NEXT: vpslld %xmm2, %xmm9, %xmm9
-; AVX1-NEXT: vblendvps %xmm15, %xmm1, %xmm9, %xmm9
-; AVX1-NEXT: vpmovsxbd %xmm13, %xmm1
-; AVX1-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[1,1,1,1]
-; AVX1-NEXT: vpmovsxbd %xmm13, %xmm13
-; AVX1-NEXT: vpcmpeqb %xmm3, %xmm14, %xmm14
-; AVX1-NEXT: vmovdqu 16(%rdi,%rcx,4), %xmm15
-; AVX1-NEXT: vpslld %xmm0, %xmm15, %xmm0
-; AVX1-NEXT: vpslld %xmm2, %xmm15, %xmm2
-; AVX1-NEXT: vpmovsxbd %xmm14, %xmm15
-; AVX1-NEXT: vpshufd {{.*#+}} xmm14 = xmm14[1,1,1,1]
-; AVX1-NEXT: vpmovsxbd %xmm14, %xmm14
-; AVX1-NEXT: vblendvps %xmm12, %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: vmovdqu 32(%rdi,%rcx,4), %xmm2
+; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[1,1,1,1]
+; AVX1-NEXT: vpmovsxbd %xmm9, %xmm9
+; AVX1-NEXT: vmovdqu (%rdi,%rcx,4), %xmm12
; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX1-NEXT: vpslld %xmm3, %xmm2, %xmm12
+; AVX1-NEXT: vpslld %xmm3, %xmm12, %xmm2
+; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX1-NEXT: vpslld %xmm4, %xmm12, %xmm12
+; AVX1-NEXT: vblendvps %xmm14, %xmm2, %xmm12, %xmm12
+; AVX1-NEXT: vmovdqu 16(%rdi,%rcx,4), %xmm2
+; AVX1-NEXT: vpslld %xmm3, %xmm2, %xmm14
; AVX1-NEXT: vpslld %xmm4, %xmm2, %xmm2
-; AVX1-NEXT: vblendvps %xmm11, %xmm12, %xmm2, %xmm2
-; AVX1-NEXT: vmovdqu 48(%rdi,%rcx,4), %xmm11
-; AVX1-NEXT: vpslld %xmm3, %xmm11, %xmm12
-; AVX1-NEXT: vpslld %xmm4, %xmm11, %xmm11
-; AVX1-NEXT: vblendvps %xmm10, %xmm12, %xmm11, %xmm10
-; AVX1-NEXT: vmovdqu 64(%rdi,%rcx,4), %xmm11
-; AVX1-NEXT: vpslld %xmm5, %xmm11, %xmm12
-; AVX1-NEXT: vpslld %xmm6, %xmm11, %xmm11
-; AVX1-NEXT: vblendvps %xmm1, %xmm12, %xmm11, %xmm1
-; AVX1-NEXT: vmovdqu 80(%rdi,%rcx,4), %xmm11
-; AVX1-NEXT: vpslld %xmm5, %xmm11, %xmm12
-; AVX1-NEXT: vpslld %xmm6, %xmm11, %xmm11
-; AVX1-NEXT: vblendvps %xmm13, %xmm12, %xmm11, %xmm11
-; AVX1-NEXT: vmovdqu 96(%rdi,%rcx,4), %xmm12
-; AVX1-NEXT: vpslld %xmm7, %xmm12, %xmm13
-; AVX1-NEXT: vpslld %xmm8, %xmm12, %xmm12
-; AVX1-NEXT: vblendvps %xmm15, %xmm13, %xmm12, %xmm12
-; AVX1-NEXT: vmovdqu 112(%rdi,%rcx,4), %xmm13
-; AVX1-NEXT: vpslld %xmm7, %xmm13, %xmm15
-; AVX1-NEXT: vpslld %xmm8, %xmm13, %xmm13
-; AVX1-NEXT: vblendvps %xmm14, %xmm15, %xmm13, %xmm13
-; AVX1-NEXT: vmovups %xmm9, (%rdi,%rcx,4)
-; AVX1-NEXT: vmovups %xmm0, 16(%rdi,%rcx,4)
-; AVX1-NEXT: vmovups %xmm2, 32(%rdi,%rcx,4)
-; AVX1-NEXT: vmovups %xmm10, 48(%rdi,%rcx,4)
-; AVX1-NEXT: vmovups %xmm1, 64(%rdi,%rcx,4)
+; AVX1-NEXT: vblendvps %xmm15, %xmm14, %xmm2, %xmm2
+; AVX1-NEXT: vmovdqu 32(%rdi,%rcx,4), %xmm14
+; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
+; AVX1-NEXT: vpslld %xmm3, %xmm14, %xmm15
+; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX1-NEXT: vpslld %xmm4, %xmm14, %xmm14
+; AVX1-NEXT: vblendvps %xmm0, %xmm15, %xmm14, %xmm0
+; AVX1-NEXT: vmovdqu 48(%rdi,%rcx,4), %xmm14
+; AVX1-NEXT: vpslld %xmm3, %xmm14, %xmm15
+; AVX1-NEXT: vpslld %xmm4, %xmm14, %xmm14
+; AVX1-NEXT: vblendvps %xmm1, %xmm15, %xmm14, %xmm1
+; AVX1-NEXT: vmovdqu 64(%rdi,%rcx,4), %xmm14
+; AVX1-NEXT: vpslld %xmm5, %xmm14, %xmm15
+; AVX1-NEXT: vpslld %xmm6, %xmm14, %xmm14
+; AVX1-NEXT: vblendvps %xmm13, %xmm15, %xmm14, %xmm13
+; AVX1-NEXT: vmovdqu 80(%rdi,%rcx,4), %xmm14
+; AVX1-NEXT: vpslld %xmm5, %xmm14, %xmm15
+; AVX1-NEXT: vpslld %xmm6, %xmm14, %xmm14
+; AVX1-NEXT: vblendvps %xmm11, %xmm15, %xmm14, %xmm11
+; AVX1-NEXT: vmovdqu 96(%rdi,%rcx,4), %xmm14
+; AVX1-NEXT: vpslld %xmm7, %xmm14, %xmm15
+; AVX1-NEXT: vpslld %xmm8, %xmm14, %xmm14
+; AVX1-NEXT: vblendvps %xmm10, %xmm15, %xmm14, %xmm10
+; AVX1-NEXT: vmovdqu 112(%rdi,%rcx,4), %xmm14
+; AVX1-NEXT: vpslld %xmm7, %xmm14, %xmm15
+; AVX1-NEXT: vpslld %xmm8, %xmm14, %xmm14
+; AVX1-NEXT: vblendvps %xmm9, %xmm15, %xmm14, %xmm9
+; AVX1-NEXT: vmovups %xmm12, (%rdi,%rcx,4)
+; AVX1-NEXT: vmovups %xmm2, 16(%rdi,%rcx,4)
+; AVX1-NEXT: vmovups %xmm0, 32(%rdi,%rcx,4)
+; AVX1-NEXT: vmovups %xmm1, 48(%rdi,%rcx,4)
+; AVX1-NEXT: vmovups %xmm13, 64(%rdi,%rcx,4)
; AVX1-NEXT: vmovups %xmm11, 80(%rdi,%rcx,4)
-; AVX1-NEXT: vmovups %xmm12, 96(%rdi,%rcx,4)
-; AVX1-NEXT: vmovups %xmm13, 112(%rdi,%rcx,4)
+; AVX1-NEXT: vmovups %xmm10, 96(%rdi,%rcx,4)
+; AVX1-NEXT: vmovups %xmm9, 112(%rdi,%rcx,4)
; AVX1-NEXT: addq $32, %rcx
; AVX1-NEXT: cmpq %rcx, %rdx
; AVX1-NEXT: jne .LBB0_4
>From fa91086bd834238587f6e193cc21722debe97d7c Mon Sep 17 00:00:00 2001
From: Rin Dobrescu <rin.dobrescu at arm.com>
Date: Wed, 8 Nov 2023 10:05:08 +0000
Subject: [PATCH 2/3] Defer IsLoopInvariantInst check until after the cheaper
register checks
---
llvm/lib/CodeGen/MachineLICM.cpp | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/CodeGen/MachineLICM.cpp b/llvm/lib/CodeGen/MachineLICM.cpp
index b32a39d239a56f3..6f72d38c095c1fa 100644
--- a/llvm/lib/CodeGen/MachineLICM.cpp
+++ b/llvm/lib/CodeGen/MachineLICM.cpp
@@ -1251,9 +1251,10 @@ bool MachineLICMBase::IsProfitableToHoist(MachineInstr &MI,
// If we have a COPY with other uses in the loop, hoist to allow the users to
// also be hoisted.
- if (MI.isCopy() && IsLoopInvariantInst(MI, CurLoop) &&
- MI.getOperand(0).isReg() && MI.getOperand(0).getReg().isVirtual() &&
- MI.getOperand(1).isReg() && MI.getOperand(1).getReg().isVirtual() &&
+ if (MI.isCopy() && MI.getOperand(0).isReg() &&
+ MI.getOperand(0).getReg().isVirtual() && MI.getOperand(1).isReg() &&
+ MI.getOperand(1).getReg().isVirtual() &&
+ IsLoopInvariantInst(MI, CurLoop) &&
any_of(MRI->use_nodbg_instructions(MI.getOperand(0).getReg()),
[&](MachineInstr &UseMI) { return CurLoop->contains(&UseMI); }))
return true;
>From 0147f506f67c303b02ff5ee7ae7b926b4c107941 Mon Sep 17 00:00:00 2001
From: Rin Dobrescu <rin.dobrescu at arm.com>
Date: Wed, 8 Nov 2023 10:59:32 +0000
Subject: [PATCH 3/3] Value capture of CurLoop
---
llvm/lib/CodeGen/MachineLICM.cpp | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/MachineLICM.cpp b/llvm/lib/CodeGen/MachineLICM.cpp
index 6f72d38c095c1fa..77a26cdfbb09031 100644
--- a/llvm/lib/CodeGen/MachineLICM.cpp
+++ b/llvm/lib/CodeGen/MachineLICM.cpp
@@ -1256,7 +1256,9 @@ bool MachineLICMBase::IsProfitableToHoist(MachineInstr &MI,
MI.getOperand(1).getReg().isVirtual() &&
IsLoopInvariantInst(MI, CurLoop) &&
any_of(MRI->use_nodbg_instructions(MI.getOperand(0).getReg()),
- [&](MachineInstr &UseMI) { return CurLoop->contains(&UseMI); }))
+ [&CurLoop](MachineInstr &UseMI) {
+ return CurLoop->contains(&UseMI);
+ }))
return true;
// High register pressure situation, only hoist if the instruction is going
More information about the cfe-commits
mailing list