[llvm] aa5d182 - [AMDGPU] Make VALU instructions defining SGPR non-ignorable (#195270)
via llvm-commits
llvm-commits at lists.llvm.org
Fri May 8 05:57:08 PDT 2026
Author: Igor Wodiany
Date: 2026-05-08T13:57:03+01:00
New Revision: aa5d182da16125577757d6c49a9fa510f680743a
URL: https://github.com/llvm/llvm-project/commit/aa5d182da16125577757d6c49a9fa510f680743a
DIFF: https://github.com/llvm/llvm-project/commit/aa5d182da16125577757d6c49a9fa510f680743a.diff
LOG: [AMDGPU] Make VALU instructions defining SGPR non-ignorable (#195270)
This fixes an issue where CSE would incorrectly eliminate an instruction
that produces a lane mask. For example, the second V_CMP_GT in the code
below cannot be replaced with %3, despite both having the same operands
as it would cause an incorrect exec mask being calculated in %6:
```
bb.1
%3:sreg_64 = V_CMP_GT_U32_e64 %0:vgpr_32, %1:sreg_32, implicit $exec
%4:sreg_64 = SI_IF_BREAK killed %3:sreg_64, %2:sreg_64, implicit-def dead $scc
SI_LOOP %4:sreg_64, %bb.1, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
S_BRANCH %bb.2
bb.2:
SI_END_CF %4:sreg_64, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
%5:sreg_64 = V_CMP_GT_U32_e64 %0:vgpr_32, %1:sreg_32, implicit $exec
%6:sreg_64 = S_AND_B64 %5:sreg_64, $exec, implicit-def $scc
```
This is submitted as a preferable solution when compared to both #194863
(large number of diffs in tests) and #193522 (more of a workaround than an
actual fix).
Added:
llvm/test/CodeGen/AMDGPU/v-cmp-cse-across-loop.mir
Modified:
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
llvm/test/CodeGen/AMDGPU/coalescer_distribute.ll
llvm/test/CodeGen/AMDGPU/collapse-endcf.ll
llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-phi-regression-issue130646-issue130119.ll
llvm/test/CodeGen/AMDGPU/licm-valu.mir
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.kill.ll
llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
llvm/test/CodeGen/AMDGPU/structurize-hoist.ll
llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll
llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 65137fa09e209..3efae655b311e 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -150,46 +150,23 @@ bool SIInstrInfo::isReMaterializableImpl(
return TargetInstrInfo::isReMaterializableImpl(MI);
}
-// Returns true if the scalar result of a VALU instruction depends on exec.
+// Returns true if the result of a VALU instruction depends on exec.
bool SIInstrInfo::resultDependsOnExec(const MachineInstr &MI) const {
- // Ignore comparisons which are only used masked with exec.
- // This allows some hoisting/sinking of VALU comparisons.
- if (MI.isCompare()) {
- const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::sdst);
- if (!Dst)
- return true;
-
- Register DstReg = Dst->getReg();
- if (!DstReg.isVirtual())
- return true;
+ assert(isVALU(MI));
- const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
- for (MachineInstr &Use : MRI.use_nodbg_instructions(DstReg)) {
- switch (Use.getOpcode()) {
- case AMDGPU::S_AND_SAVEEXEC_B32:
- case AMDGPU::S_AND_SAVEEXEC_B64:
- break;
- case AMDGPU::S_AND_B32:
- case AMDGPU::S_AND_B64:
- if (!Use.readsRegister(AMDGPU::EXEC, /*TRI=*/nullptr))
- return true;
- break;
- default:
- return true;
- }
- }
- return false;
- }
+ // If it is convergent it depends on EXEC.
+ if (MI.isConvergent())
+ return true;
- // If it is not convergent it does not depend on EXEC.
- if (!MI.isConvergent())
- return false;
+ // If it defines SGPR it depends on EXEC
+ const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
+ for (const MachineOperand &Def : MI.defs()) {
+ if (!Def.isReg())
+ continue;
- switch (MI.getOpcode()) {
- default:
- break;
- case AMDGPU::V_READFIRSTLANE_B32:
- return true;
+ Register Reg = Def.getReg();
+ if (Reg && RI.isSGPRReg(MRI, Reg))
+ return true;
}
return false;
diff --git a/llvm/test/CodeGen/AMDGPU/coalescer_distribute.ll b/llvm/test/CodeGen/AMDGPU/coalescer_distribute.ll
index d07cc84865bea..94085ad99ccda 100644
--- a/llvm/test/CodeGen/AMDGPU/coalescer_distribute.ll
+++ b/llvm/test/CodeGen/AMDGPU/coalescer_distribute.ll
@@ -18,11 +18,9 @@ define amdgpu_kernel void @hoge(i1 %c0, i1 %c1, i1 %c2, i1 %c3, i1 %c4) {
; CHECK-NEXT: s_bitcmp1_b32 s2, 24
; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
; CHECK-NEXT: s_xor_b64 s[0:1], s[0:1], -1
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; CHECK-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0
; CHECK-NEXT: .LBB0_1: ; %bb25
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: s_and_b64 vcc, exec, s[0:1]
+; CHECK-NEXT: s_andn2_b64 vcc, exec, s[0:1]
; CHECK-NEXT: s_cbranch_vccnz .LBB0_1
; CHECK-NEXT: ; %bb.2: ; %bb30
; CHECK-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll b/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll
index 2f287d269e4d8..c554e26a43965 100644
--- a/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll
+++ b/llvm/test/CodeGen/AMDGPU/collapse-endcf.ll
@@ -14,26 +14,26 @@ define amdgpu_kernel void @simple_nested_if(ptr addrspace(1) nocapture %arg) {
; GCN-NEXT: s_cbranch_execz .LBB0_3
; GCN-NEXT: ; %bb.1: ; %bb.outer.then
; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: v_lshlrev_b32_e32 v1, 2, v0
-; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: v_lshlrev_b32_e32 v3, 2, v0
+; GCN-NEXT: v_mov_b32_e32 v4, 0
; GCN-NEXT: s_mov_b32 s3, 0xf000
; GCN-NEXT: s_mov_b32 s2, 0
-; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 2, v0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: buffer_store_dword v2, v[1:2], s[0:3], 0 addr64
+; GCN-NEXT: v_mov_b32_e32 v2, s1
+; GCN-NEXT: v_add_i32_e32 v1, vcc, s0, v3
+; GCN-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc
+; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 2, v0
+; GCN-NEXT: buffer_store_dword v4, v[3:4], s[0:3], 0 addr64
; GCN-NEXT: s_and_b64 exec, exec, vcc
; GCN-NEXT: s_cbranch_execz .LBB0_3
; GCN-NEXT: ; %bb.2: ; %bb.inner.then
-; GCN-NEXT: s_waitcnt expcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v2, s1
-; GCN-NEXT: v_add_i32_e32 v0, vcc, s0, v1
-; GCN-NEXT: v_addc_u32_e32 v1, vcc, 0, v2, vcc
; GCN-NEXT: s_mov_b32 s0, s2
; GCN-NEXT: s_mov_b32 s1, s2
-; GCN-NEXT: v_mov_b32_e32 v2, 1
-; GCN-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:4
+; GCN-NEXT: v_mov_b32_e32 v0, 1
+; GCN-NEXT: buffer_store_dword v0, v[1:2], s[0:3], 0 addr64 offset:4
; GCN-NEXT: .LBB0_3: ; %bb.outer.end
; GCN-NEXT: s_or_b64 exec, exec, s[6:7]
+; GCN-NEXT: s_waitcnt expcnt(0)
; GCN-NEXT: v_mov_b32_e32 v0, 3
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: s_mov_b32 m0, -1
@@ -376,18 +376,18 @@ define amdgpu_kernel void @nested_if_if_else(ptr addrspace(1) nocapture %arg) {
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: s_mov_b32 s3, 0xf000
; GCN-NEXT: s_mov_b32 s2, 0
-; GCN-NEXT: v_cmp_lt_u32_e32 vcc, 1, v0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: buffer_store_dword v2, v[1:2], s[0:3], 0 addr64
-; GCN-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GCN-NEXT: s_cbranch_execz .LBB2_5
-; GCN-NEXT: ; %bb.1: ; %bb.outer.then
; GCN-NEXT: v_mov_b32_e32 v4, s1
; GCN-NEXT: v_add_i32_e32 v3, vcc, s0, v1
; GCN-NEXT: v_addc_u32_e32 v4, vcc, 0, v4, vcc
-; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 2, v0
+; GCN-NEXT: v_cmp_lt_u32_e32 vcc, 1, v0
+; GCN-NEXT: buffer_store_dword v2, v[1:2], s[0:3], 0 addr64
; GCN-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GCN-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GCN-NEXT: s_cbranch_execz .LBB2_5
+; GCN-NEXT: ; %bb.1: ; %bb.outer.then
+; GCN-NEXT: v_cmp_ne_u32_e32 vcc, 2, v0
+; GCN-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GCN-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GCN-NEXT: s_cbranch_execz .LBB2_3
; GCN-NEXT: ; %bb.2: ; %bb.else
; GCN-NEXT: s_mov_b32 s6, 0
@@ -398,7 +398,7 @@ define amdgpu_kernel void @nested_if_if_else(ptr addrspace(1) nocapture %arg) {
; GCN-NEXT: buffer_store_dword v0, v[3:4], s[4:7], 0 addr64 offset:8
; GCN-NEXT: ; implicit-def: $vgpr3_vgpr4
; GCN-NEXT: .LBB2_3: ; %Flow
-; GCN-NEXT: s_andn2_saveexec_b64 s[0:1], s[0:1]
+; GCN-NEXT: s_andn2_saveexec_b64 s[2:3], s[2:3]
; GCN-NEXT: s_cbranch_execz .LBB2_5
; GCN-NEXT: ; %bb.4: ; %bb.then
; GCN-NEXT: s_mov_b32 s6, 0
@@ -409,7 +409,7 @@ define amdgpu_kernel void @nested_if_if_else(ptr addrspace(1) nocapture %arg) {
; GCN-NEXT: v_mov_b32_e32 v0, 1
; GCN-NEXT: buffer_store_dword v0, v[3:4], s[4:7], 0 addr64 offset:4
; GCN-NEXT: .LBB2_5: ; %bb.outer.end
-; GCN-NEXT: s_or_b64 exec, exec, s[2:3]
+; GCN-NEXT: s_or_b64 exec, exec, s[0:1]
; GCN-NEXT: s_waitcnt expcnt(0)
; GCN-NEXT: v_mov_b32_e32 v0, 3
; GCN-NEXT: s_mov_b32 m0, -1
diff --git a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-phi-regression-issue130646-issue130119.ll b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-phi-regression-issue130646-issue130119.ll
index d03d53a8cbbaa..0c25ffd80dbad 100644
--- a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-phi-regression-issue130646-issue130119.ll
+++ b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-phi-regression-issue130646-issue130119.ll
@@ -73,43 +73,41 @@ define amdgpu_cs void @issue130119(i1 %arg) {
; CHECK-NEXT: v_and_b32_e32 v0, 1, v0
; CHECK-NEXT: v_cmp_eq_u32_e64 s[0:1], 1, v0
; CHECK-NEXT: s_mov_b32 s16, 0
-; CHECK-NEXT: s_mov_b64 s[4:5], 0
+; CHECK-NEXT: s_mov_b64 s[2:3], 0
; CHECK-NEXT: s_branch .LBB1_2
; CHECK-NEXT: .LBB1_1: ; %Flow2
; CHECK-NEXT: ; in Loop: Header=BB1_2 Depth=1
; CHECK-NEXT: s_or_b64 exec, exec, s[6:7]
-; CHECK-NEXT: s_and_b64 s[2:3], exec, s[2:3]
-; CHECK-NEXT: s_or_b64 s[4:5], s[2:3], s[4:5]
-; CHECK-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; CHECK-NEXT: s_and_b64 s[4:5], exec, s[4:5]
+; CHECK-NEXT: s_or_b64 s[2:3], s[4:5], s[2:3]
+; CHECK-NEXT: s_andn2_b64 exec, exec, s[2:3]
; CHECK-NEXT: s_cbranch_execz .LBB1_10
; CHECK-NEXT: .LBB1_2: ; %bb1
; CHECK-NEXT: ; =>This Loop Header: Depth=1
; CHECK-NEXT: ; Child Loop BB1_4 Depth 2
-; CHECK-NEXT: s_and_b32 s2, s16, 1
-; CHECK-NEXT: s_cmp_eq_u32 s2, 0
-; CHECK-NEXT: s_cselect_b64 s[6:7], -1, 0
-; CHECK-NEXT: s_cmp_eq_u32 s2, 1
-; CHECK-NEXT: s_cselect_b64 s[2:3], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[2:3]
-; CHECK-NEXT: v_cmp_ne_u32_e64 s[2:3], 1, v0
+; CHECK-NEXT: s_and_b32 s6, s16, 1
+; CHECK-NEXT: s_cmp_eq_u32 s6, 0
+; CHECK-NEXT: s_cselect_b64 s[4:5], -1, 0
+; CHECK-NEXT: s_cmp_eq_u32 s6, 1
+; CHECK-NEXT: s_cselect_b64 s[8:9], -1, 0
; CHECK-NEXT: s_mov_b64 s[10:11], 0
-; CHECK-NEXT: ; implicit-def: $sgpr8_sgpr9
+; CHECK-NEXT: ; implicit-def: $sgpr6_sgpr7
; CHECK-NEXT: s_branch .LBB1_4
; CHECK-NEXT: .LBB1_3: ; %Flow1
; CHECK-NEXT: ; in Loop: Header=BB1_4 Depth=2
; CHECK-NEXT: s_xor_b64 s[14:15], s[14:15], -1
; CHECK-NEXT: s_and_b64 s[12:13], exec, s[12:13]
; CHECK-NEXT: s_or_b64 s[10:11], s[12:13], s[10:11]
-; CHECK-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; CHECK-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
; CHECK-NEXT: s_and_b64 s[12:13], s[14:15], exec
-; CHECK-NEXT: s_or_b64 s[8:9], s[8:9], s[12:13]
+; CHECK-NEXT: s_or_b64 s[6:7], s[6:7], s[12:13]
; CHECK-NEXT: s_andn2_b64 exec, exec, s[10:11]
; CHECK-NEXT: s_cbranch_execz .LBB1_8
; CHECK-NEXT: .LBB1_4: ; %bb3
; CHECK-NEXT: ; Parent Loop BB1_2 Depth=1
; CHECK-NEXT: ; => This Inner Loop Header: Depth=2
-; CHECK-NEXT: s_and_b64 vcc, exec, s[2:3]
-; CHECK-NEXT: s_mov_b64 s[14:15], s[6:7]
+; CHECK-NEXT: s_andn2_b64 vcc, exec, s[8:9]
+; CHECK-NEXT: s_mov_b64 s[14:15], s[4:5]
; CHECK-NEXT: s_cbranch_vccnz .LBB1_6
; CHECK-NEXT: ; %bb.5: ; %bb7
; CHECK-NEXT: ; in Loop: Header=BB1_4 Depth=2
@@ -128,14 +126,14 @@ define amdgpu_cs void @issue130119(i1 %arg) {
; CHECK-NEXT: .LBB1_8: ; %loop.exit.guard
; CHECK-NEXT: ; in Loop: Header=BB1_2 Depth=1
; CHECK-NEXT: s_or_b64 exec, exec, s[10:11]
-; CHECK-NEXT: s_mov_b64 s[2:3], -1
-; CHECK-NEXT: s_and_saveexec_b64 s[6:7], s[8:9]
-; CHECK-NEXT: s_xor_b64 s[6:7], exec, s[6:7]
+; CHECK-NEXT: s_mov_b64 s[4:5], -1
+; CHECK-NEXT: s_and_saveexec_b64 s[8:9], s[6:7]
+; CHECK-NEXT: s_xor_b64 s[6:7], exec, s[8:9]
; CHECK-NEXT: s_cbranch_execz .LBB1_1
; CHECK-NEXT: ; %bb.9: ; %bb10
; CHECK-NEXT: ; in Loop: Header=BB1_2 Depth=1
; CHECK-NEXT: s_or_b32 s16, s16, 1
-; CHECK-NEXT: s_xor_b64 s[2:3], exec, -1
+; CHECK-NEXT: s_xor_b64 s[4:5], exec, -1
; CHECK-NEXT: s_branch .LBB1_1
; CHECK-NEXT: .LBB1_10: ; %DummyReturnBlock
; CHECK-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/licm-valu.mir b/llvm/test/CodeGen/AMDGPU/licm-valu.mir
index 0020e89580a14..a27153867408b 100644
--- a/llvm/test/CodeGen/AMDGPU/licm-valu.mir
+++ b/llvm/test/CodeGen/AMDGPU/licm-valu.mir
@@ -74,12 +74,12 @@ body: |
; GCN: bb.0:
; GCN-NEXT: successors: %bb.1(0x80000000)
; GCN-NEXT: {{ $}}
- ; GCN-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 1, 2, implicit $exec
; GCN-NEXT: S_BRANCH %bb.1
; GCN-NEXT: {{ $}}
; GCN-NEXT: bb.1:
; GCN-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 1, 2, implicit $exec
; GCN-NEXT: $exec = S_AND_B64 $exec, [[V_CMP_EQ_U32_e64_]], implicit-def $scc
; GCN-NEXT: S_CBRANCH_EXECNZ %bb.1, implicit $exec
; GCN-NEXT: S_BRANCH %bb.2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.kill.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.kill.ll
index af81d95973452..fe7adab59a23c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.kill.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.kill.ll
@@ -1527,14 +1527,12 @@ define amdgpu_ps void @kill_with_loop_exit(float inreg %inp0, float inreg %inp1,
; SI-NEXT: v_mov_b32_e32 v0, 1.0
; SI-NEXT: s_cbranch_vccnz .LBB26_5
; SI-NEXT: ; %bb.1: ; %.preheader1.preheader
-; SI-NEXT: v_cmp_ngt_f32_e64 s[0:1], s6, 0
-; SI-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[0:1]
; SI-NEXT: s_mov_b64 s[2:3], exec
; SI-NEXT: v_mov_b32_e32 v0, 0x3fc00000
-; SI-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v1
+; SI-NEXT: v_cmp_ngt_f32_e64 s[0:1], s6, 0
; SI-NEXT: .LBB26_2: ; %bb
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: s_and_b64 vcc, exec, s[0:1]
+; SI-NEXT: s_andn2_b64 vcc, exec, s[0:1]
; SI-NEXT: v_add_f32_e32 v0, 0x3e800000, v0
; SI-NEXT: s_cbranch_vccnz .LBB26_2
; SI-NEXT: ; %bb.3: ; %bb33
@@ -1559,15 +1557,13 @@ define amdgpu_ps void @kill_with_loop_exit(float inreg %inp0, float inreg %inp1,
; GFX10-NEXT: s_and_b64 vcc, exec, s[0:1]
; GFX10-NEXT: s_cbranch_vccnz .LBB26_5
; GFX10-NEXT: ; %bb.1: ; %.preheader1.preheader
-; GFX10-NEXT: v_cmp_ngt_f32_e64 s[0:1], s6, 0
; GFX10-NEXT: v_mov_b32_e32 v0, 0x3fc00000
+; GFX10-NEXT: v_cmp_ngt_f32_e64 s[0:1], s6, 0
; GFX10-NEXT: s_mov_b64 s[2:3], exec
-; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[0:1]
-; GFX10-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v1
; GFX10-NEXT: .LBB26_2: ; %bb
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_add_f32_e32 v0, 0x3e800000, v0
-; GFX10-NEXT: s_and_b64 vcc, exec, s[0:1]
+; GFX10-NEXT: s_andn2_b64 vcc, exec, s[0:1]
; GFX10-NEXT: s_cbranch_vccnz .LBB26_2
; GFX10-NEXT: ; %bb.3: ; %bb33
; GFX10-NEXT: s_andn2_b64 s[2:3], s[2:3], exec
@@ -1591,16 +1587,13 @@ define amdgpu_ps void @kill_with_loop_exit(float inreg %inp0, float inreg %inp1,
; GFX11-NEXT: s_and_b64 vcc, exec, s[0:1]
; GFX11-NEXT: s_cbranch_vccnz .LBB26_5
; GFX11-NEXT: ; %bb.1: ; %.preheader1.preheader
-; GFX11-NEXT: v_cmp_ngt_f32_e64 s[0:1], s6, 0
; GFX11-NEXT: v_mov_b32_e32 v0, 0x3fc00000
+; GFX11-NEXT: v_cmp_ngt_f32_e64 s[0:1], s6, 0
; GFX11-NEXT: s_mov_b64 s[2:3], exec
-; GFX11-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[0:1]
-; GFX11-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v1
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_sdst(0)
; GFX11-NEXT: .LBB26_2: ; %bb
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: v_add_f32_e32 v0, 0x3e800000, v0
-; GFX11-NEXT: s_and_b64 vcc, exec, s[0:1]
+; GFX11-NEXT: s_and_not1_b64 vcc, exec, s[0:1]
; GFX11-NEXT: s_cbranch_vccnz .LBB26_2
; GFX11-NEXT: ; %bb.3: ; %bb33
; GFX11-NEXT: s_and_not1_b64 s[2:3], s[2:3], exec
@@ -1630,12 +1623,10 @@ define amdgpu_ps void @kill_with_loop_exit(float inreg %inp0, float inreg %inp1,
; GFX12-NEXT: s_mov_b64 s[2:3], exec
; GFX12-NEXT: s_mov_b32 s4, 0x3fc00000
; GFX12-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX12-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; GFX12-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0
; GFX12-NEXT: .LBB26_2: ; %bb
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_add_f32 s4, s4, 0x3e800000
-; GFX12-NEXT: s_and_b64 vcc, exec, s[0:1]
+; GFX12-NEXT: s_and_not1_b64 vcc, exec, s[0:1]
; GFX12-NEXT: s_cbranch_vccnz .LBB26_2
; GFX12-NEXT: ; %bb.3: ; %bb33
; GFX12-NEXT: s_and_not1_b64 s[2:3], s[2:3], exec
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
index 899f924ac06bd..b0da72184f050 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
@@ -225,21 +225,23 @@ define float @local_atomic_fadd_ret_f32__offset(ptr addrspace(3) %ptr) nounwind
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v0, v1
+; GFX6-NEXT: ds_read_b32 v1, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v2, v0
-; GFX6-NEXT: v_add_f32_e32 v0, 4.0, v2
-; GFX6-NEXT: ds_cmpst_rtn_b32 v0, v1, v2, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, v1
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffc, v0
+; GFX6-NEXT: v_add_f32_e32 v3, 4.0, v2
+; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v1, v2, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB1_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX6-NEXT: v_mov_b32_e32 v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(3) %ptr, i32 16383
%result = atomicrmw fadd ptr addrspace(3) %gep, float 4.0 seq_cst
@@ -451,15 +453,16 @@ define void @local_atomic_fadd_noret_f32__offset(ptr addrspace(3) %ptr) nounwind
; GFX6-LABEL: local_atomic_fadd_noret_f32__offset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffc, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v1, v0
+; GFX6-NEXT: ds_read_b32 v1, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_add_f32_e32 v2, 4.0, v1
-; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 0xfffc, v0
+; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v3, v1, v2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -837,9 +840,10 @@ define double @local_atomic_fadd_ret_f64__offset(ptr addrspace(3) %ptr) nounwind
; GFX6-LABEL: local_atomic_fadd_ret_f64__offset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfff8, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, v0
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfff8, v2
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b64 v[0:1], v2
+; GFX6-NEXT: ds_read_b64 v[0:1], v0
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -847,7 +851,8 @@ define double @local_atomic_fadd_ret_f64__offset(ptr addrspace(3) %ptr) nounwind
; GFX6-NEXT: v_mov_b32_e32 v4, v1
; GFX6-NEXT: v_mov_b32_e32 v3, v0
; GFX6-NEXT: v_add_f64 v[0:1], v[3:4], 4.0
-; GFX6-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1]
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 0xfff8, v2
+; GFX6-NEXT: ds_cmpst_rtn_b64 v[0:1], v5, v[3:4], v[0:1]
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[3:4]
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -1203,20 +1208,21 @@ define void @local_atomic_fadd_noret_f64__offset(ptr addrspace(3) %ptr) nounwind
; GFX6-LABEL: local_atomic_fadd_noret_f64__offset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfff8, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfff8, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b64 v[0:1], v2
+; GFX6-NEXT: ds_read_b64 v[1:2], v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_add_f64 v[3:4], v[0:1], 4.0
-; GFX6-NEXT: ds_cmpst_rtn_b64 v[3:4], v2, v[0:1], v[3:4]
+; GFX6-NEXT: v_add_f64 v[3:4], v[1:2], 4.0
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 0xfff8, v0
+; GFX6-NEXT: ds_cmpst_rtn_b64 v[3:4], v5, v[1:2], v[3:4]
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v0, v3
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[1:2]
+; GFX6-NEXT: v_mov_b32_e32 v1, v3
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
+; GFX6-NEXT: v_mov_b32_e32 v2, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB7_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3025,25 +3031,27 @@ define half @local_atomic_fadd_ret_f16__offset__align4(ptr addrspace(3) %ptr) no
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v0, v1
+; GFX6-NEXT: ds_read_b32 v1, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v2, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v2
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_add_f32_e32 v0, 4.0, v0
-; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX6-NEXT: ds_cmpst_rtn_b32 v0, v1, v2, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, v1
+; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v2
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 0xfffe, v0
+; GFX6-NEXT: v_add_f32_e32 v1, 4.0, v1
+; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v3, v2, v1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB12_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX6-NEXT: v_mov_b32_e32 v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(3) %ptr, i32 32767
%result = atomicrmw fadd ptr addrspace(3) %gep, half 4.0 seq_cst, align 4
@@ -3306,19 +3314,20 @@ define void @local_atomic_fadd_noret_f16__offset__align4(ptr addrspace(3) %ptr)
; GFX6-LABEL: local_atomic_fadd_noret_f16__offset__align4:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffe, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v1, v0
+; GFX6-NEXT: ds_read_b32 v1, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v1
; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffe, v0
; GFX6-NEXT: v_add_f32_e32 v2, 4.0, v2
; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
+; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v4, v1, v2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -5491,25 +5500,27 @@ define bfloat @local_atomic_fadd_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v0, v1
+; GFX6-NEXT: ds_read_b32 v1, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v2, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX6-NEXT: v_add_f32_e32 v0, 4.0, v0
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX6-NEXT: ds_cmpst_rtn_b32 v0, v1, v2, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX6-NEXT: v_add_f32_e32 v3, 4.0, v3
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
+; GFX6-NEXT: v_or_b32_e32 v3, v4, v3
+; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v1, v2, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB18_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX6-NEXT: v_mov_b32_e32 v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(3) %ptr, i32 32767
%result = atomicrmw fadd ptr addrspace(3) %gep, bfloat 4.0 seq_cst, align 4
@@ -5848,9 +5859,9 @@ define void @local_atomic_fadd_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX6-LABEL: local_atomic_fadd_noret_bf16__offset__align4:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffe, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v1, v0
+; GFX6-NEXT: ds_read_b32 v1, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -5859,8 +5870,9 @@ define void @local_atomic_fadd_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX6-NEXT: v_add_f32_e32 v2, 4.0, v2
; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffe, v0
; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
+; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v4, v1, v2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -6270,9 +6282,9 @@ define <2 x half> @local_atomic_fadd_ret_v2f16__offset(ptr addrspace(3) %ptr, <2
; GFX6-LABEL: local_atomic_fadd_ret_v2f16__offset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffc, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v3, v0
+; GFX6-NEXT: ds_read_b32 v3, v2
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
@@ -6290,9 +6302,10 @@ define <2 x half> @local_atomic_fadd_ret_v2f16__offset(ptr addrspace(3) %ptr, <2
; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
; GFX6-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX6-NEXT: v_add_i32_e32 v7, vcc, 0xfffc, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX6-NEXT: v_or_b32_e32 v3, v6, v3
-; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3
+; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v7, v4, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -6679,9 +6692,9 @@ define void @local_atomic_fadd_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
; GFX6-LABEL: local_atomic_fadd_noret_v2f16__offset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffc, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v3, v0
+; GFX6-NEXT: ds_read_b32 v3, v2
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
@@ -6699,9 +6712,10 @@ define void @local_atomic_fadd_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
; GFX6-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX6-NEXT: v_add_i32_e32 v7, vcc, 0xfffc, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX6-NEXT: v_or_b32_e32 v3, v6, v3
-; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3
+; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v7, v4, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -7399,9 +7413,9 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX6-LABEL: local_atomic_fadd_ret_v2bf16__offset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffc, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v2, v0
+; GFX6-NEXT: ds_read_b32 v2, v2
; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
@@ -7416,15 +7430,16 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_add_f32_e32 v5, v5, v1
+; GFX6-NEXT: v_add_f32_e32 v7, v7, v2
+; GFX6-NEXT: v_add_f32_e32 v6, v6, v1
; GFX6-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 0xfffc, v0
+; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
+; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v5, v3, v4
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
@@ -8093,9 +8108,9 @@ define void @local_atomic_fadd_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX6-LABEL: local_atomic_fadd_noret_v2bf16__ofset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffc, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v2, v0
+; GFX6-NEXT: ds_read_b32 v2, v2
; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
@@ -8110,15 +8125,16 @@ define void @local_atomic_fadd_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_add_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_add_f32_e32 v5, v5, v1
+; GFX6-NEXT: v_add_f32_e32 v7, v7, v2
+; GFX6-NEXT: v_add_f32_e32 v6, v6, v1
; GFX6-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 0xfffc, v0
+; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
+; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v5, v3, v4
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
index b7ddb8321c68e..43d77d0afa8c6 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
@@ -2616,25 +2616,27 @@ define half @local_atomic_fmax_ret_f16__offset__align4(ptr addrspace(3) %ptr) no
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v0, v1
+; GFX6-NEXT: ds_read_b32 v1, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v2, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v2
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_max_f32_e32 v0, 4.0, v0
-; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX6-NEXT: ds_cmpst_rtn_b32 v0, v1, v2, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, v1
+; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v2
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 0xfffe, v0
+; GFX6-NEXT: v_max_f32_e32 v1, 4.0, v1
+; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v3, v2, v1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB12_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX6-NEXT: v_mov_b32_e32 v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(3) %ptr, i32 32767
%result = atomicrmw fmax ptr addrspace(3) %gep, half 4.0 seq_cst, align 4
@@ -2908,19 +2910,20 @@ define void @local_atomic_fmax_noret_f16__offset__align4(ptr addrspace(3) %ptr)
; GFX6-LABEL: local_atomic_fmax_noret_f16__offset__align4:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffe, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v1, v0
+; GFX6-NEXT: ds_read_b32 v1, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v1
; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffe, v0
; GFX6-NEXT: v_max_f32_e32 v2, 4.0, v2
; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
+; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v4, v1, v2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -5102,26 +5105,28 @@ define bfloat @local_atomic_fmax_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v0, v1
+; GFX6-NEXT: ds_read_b32 v1, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v2, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: v_max_f32_e32 v0, 4.0, v0
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX6-NEXT: ds_cmpst_rtn_b32 v0, v1, v2, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX6-NEXT: v_max_f32_e32 v3, 4.0, v3
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
+; GFX6-NEXT: v_or_b32_e32 v3, v4, v3
+; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v1, v2, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB18_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX6-NEXT: v_mov_b32_e32 v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(3) %ptr, i32 32767
%result = atomicrmw fmax ptr addrspace(3) %gep, bfloat 4.0 seq_cst, align 4
@@ -5461,9 +5466,9 @@ define void @local_atomic_fmax_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX6-LABEL: local_atomic_fmax_noret_bf16__offset__align4:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffe, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v1, v0
+; GFX6-NEXT: ds_read_b32 v1, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -5473,8 +5478,9 @@ define void @local_atomic_fmax_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX6-NEXT: v_max_f32_e32 v2, 4.0, v2
; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffe, v0
; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
+; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v4, v1, v2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -5980,9 +5986,9 @@ define <2 x half> @local_atomic_fmax_ret_v2f16__offset(ptr addrspace(3) %ptr, <2
; GFX6-LABEL: local_atomic_fmax_ret_v2f16__offset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffc, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v3, v0
+; GFX6-NEXT: ds_read_b32 v3, v2
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
@@ -6000,9 +6006,10 @@ define <2 x half> @local_atomic_fmax_ret_v2f16__offset(ptr addrspace(3) %ptr, <2
; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
; GFX6-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX6-NEXT: v_add_i32_e32 v7, vcc, 0xfffc, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX6-NEXT: v_or_b32_e32 v3, v6, v3
-; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3
+; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v7, v4, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -6483,9 +6490,9 @@ define void @local_atomic_fmax_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
; GFX6-LABEL: local_atomic_fmax_noret_v2f16__offset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffc, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v3, v0
+; GFX6-NEXT: ds_read_b32 v3, v2
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
@@ -6503,9 +6510,10 @@ define void @local_atomic_fmax_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
; GFX6-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX6-NEXT: v_add_i32_e32 v7, vcc, 0xfffc, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX6-NEXT: v_or_b32_e32 v3, v6, v3
-; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3
+; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v7, v4, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -7449,9 +7457,9 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX6-LABEL: local_atomic_fmax_ret_v2bf16__offset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffc, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v2, v0
+; GFX6-NEXT: ds_read_b32 v2, v2
; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v1
; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v5
@@ -7466,15 +7474,16 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
+; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_max_f32_e32 v5, v5, v1
-; GFX6-NEXT: v_max_f32_e32 v6, v6, v2
+; GFX6-NEXT: v_max_f32_e32 v6, v6, v1
+; GFX6-NEXT: v_max_f32_e32 v7, v7, v2
; GFX6-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v5
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v0, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 0xfffc, v0
+; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
+; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v5, v3, v4
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
@@ -8375,9 +8384,9 @@ define void @local_atomic_fmax_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX6-LABEL: local_atomic_fmax_noret_v2bf16__ofset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffc, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v2, v0
+; GFX6-NEXT: ds_read_b32 v2, v2
; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v1
; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v5
@@ -8392,15 +8401,16 @@ define void @local_atomic_fmax_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
+; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_max_f32_e32 v5, v5, v1
-; GFX6-NEXT: v_max_f32_e32 v6, v6, v2
+; GFX6-NEXT: v_max_f32_e32 v6, v6, v1
+; GFX6-NEXT: v_max_f32_e32 v7, v7, v2
; GFX6-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v5
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v0, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 0xfffc, v0
+; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
+; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v5, v3, v4
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
index 81dcd95a64bf6..1759a212c6ea3 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
@@ -2616,25 +2616,27 @@ define half @local_atomic_fmin_ret_f16__offset__align4(ptr addrspace(3) %ptr) no
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v0, v1
+; GFX6-NEXT: ds_read_b32 v1, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v2, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v2
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_min_f32_e32 v0, 4.0, v0
-; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX6-NEXT: ds_cmpst_rtn_b32 v0, v1, v2, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, v1
+; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v2
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 0xfffe, v0
+; GFX6-NEXT: v_min_f32_e32 v1, 4.0, v1
+; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v3, v2, v1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB12_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX6-NEXT: v_mov_b32_e32 v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(3) %ptr, i32 32767
%result = atomicrmw fmin ptr addrspace(3) %gep, half 4.0 seq_cst, align 4
@@ -2908,19 +2910,20 @@ define void @local_atomic_fmin_noret_f16__offset__align4(ptr addrspace(3) %ptr)
; GFX6-LABEL: local_atomic_fmin_noret_f16__offset__align4:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffe, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v1, v0
+; GFX6-NEXT: ds_read_b32 v1, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v1
; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffe, v0
; GFX6-NEXT: v_min_f32_e32 v2, 4.0, v2
; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
+; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v4, v1, v2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -5102,26 +5105,28 @@ define bfloat @local_atomic_fmin_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v0, v1
+; GFX6-NEXT: ds_read_b32 v1, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v2, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: v_min_f32_e32 v0, 4.0, v0
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX6-NEXT: ds_cmpst_rtn_b32 v0, v1, v2, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX6-NEXT: v_min_f32_e32 v3, 4.0, v3
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
+; GFX6-NEXT: v_or_b32_e32 v3, v4, v3
+; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v1, v2, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB18_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX6-NEXT: v_mov_b32_e32 v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(3) %ptr, i32 32767
%result = atomicrmw fmin ptr addrspace(3) %gep, bfloat 4.0 seq_cst, align 4
@@ -5461,9 +5466,9 @@ define void @local_atomic_fmin_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX6-LABEL: local_atomic_fmin_noret_bf16__offset__align4:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffe, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v1, v0
+; GFX6-NEXT: ds_read_b32 v1, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -5473,8 +5478,9 @@ define void @local_atomic_fmin_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX6-NEXT: v_min_f32_e32 v2, 4.0, v2
; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffe, v0
; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
+; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v4, v1, v2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -5980,9 +5986,9 @@ define <2 x half> @local_atomic_fmin_ret_v2f16__offset(ptr addrspace(3) %ptr, <2
; GFX6-LABEL: local_atomic_fmin_ret_v2f16__offset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffc, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v3, v0
+; GFX6-NEXT: ds_read_b32 v3, v2
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
@@ -6000,9 +6006,10 @@ define <2 x half> @local_atomic_fmin_ret_v2f16__offset(ptr addrspace(3) %ptr, <2
; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
; GFX6-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX6-NEXT: v_add_i32_e32 v7, vcc, 0xfffc, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX6-NEXT: v_or_b32_e32 v3, v6, v3
-; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3
+; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v7, v4, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -6483,9 +6490,9 @@ define void @local_atomic_fmin_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
; GFX6-LABEL: local_atomic_fmin_noret_v2f16__offset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffc, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v3, v0
+; GFX6-NEXT: ds_read_b32 v3, v2
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
@@ -6503,9 +6510,10 @@ define void @local_atomic_fmin_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
; GFX6-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX6-NEXT: v_add_i32_e32 v7, vcc, 0xfffc, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX6-NEXT: v_or_b32_e32 v3, v6, v3
-; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3
+; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v7, v4, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -7449,9 +7457,9 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX6-LABEL: local_atomic_fmin_ret_v2bf16__offset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffc, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v2, v0
+; GFX6-NEXT: ds_read_b32 v2, v2
; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v1
; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v5
@@ -7466,15 +7474,16 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
+; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_min_f32_e32 v5, v5, v1
-; GFX6-NEXT: v_min_f32_e32 v6, v6, v2
+; GFX6-NEXT: v_min_f32_e32 v6, v6, v1
+; GFX6-NEXT: v_min_f32_e32 v7, v7, v2
; GFX6-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v5
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v0, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 0xfffc, v0
+; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
+; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v5, v3, v4
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
@@ -8375,9 +8384,9 @@ define void @local_atomic_fmin_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX6-LABEL: local_atomic_fmin_noret_v2bf16__ofset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffc, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v2, v0
+; GFX6-NEXT: ds_read_b32 v2, v2
; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v1
; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v5
@@ -8392,15 +8401,16 @@ define void @local_atomic_fmin_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
+; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_min_f32_e32 v5, v5, v1
-; GFX6-NEXT: v_min_f32_e32 v6, v6, v2
+; GFX6-NEXT: v_min_f32_e32 v6, v6, v1
+; GFX6-NEXT: v_min_f32_e32 v7, v7, v2
; GFX6-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v5
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
-; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v0, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 0xfffc, v0
+; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, 16
+; GFX6-NEXT: ds_cmpst_rtn_b32 v5, v5, v3, v4
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
index 7bb36262cf389..5feb76edf3d17 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
@@ -417,21 +417,23 @@ define float @local_atomic_fsub_ret_f32__offset(ptr addrspace(3) %ptr) nounwind
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v0, v1
+; GFX6-NEXT: ds_read_b32 v1, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v2, v0
-; GFX6-NEXT: v_add_f32_e32 v0, -4.0, v2
-; GFX6-NEXT: ds_cmpst_rtn_b32 v0, v1, v2, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, v1
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffc, v0
+; GFX6-NEXT: v_add_f32_e32 v3, -4.0, v2
+; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v1, v2, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB1_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX6-NEXT: v_mov_b32_e32 v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr float, ptr addrspace(3) %ptr, i32 16383
%result = atomicrmw fsub ptr addrspace(3) %gep, float 4.0 seq_cst
@@ -817,15 +819,16 @@ define void @local_atomic_fsub_noret_f32__offset(ptr addrspace(3) %ptr) nounwind
; GFX6-LABEL: local_atomic_fsub_noret_f32__offset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffc, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v1, v0
+; GFX6-NEXT: ds_read_b32 v1, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_add_f32_e32 v2, -4.0, v1
-; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 0xfffc, v0
+; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v3, v1, v2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -1253,9 +1256,10 @@ define double @local_atomic_fsub_ret_f64__offset(ptr addrspace(3) %ptr) nounwind
; GFX6-LABEL: local_atomic_fsub_ret_f64__offset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfff8, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, v0
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfff8, v2
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b64 v[0:1], v2
+; GFX6-NEXT: ds_read_b64 v[0:1], v0
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -1263,7 +1267,8 @@ define double @local_atomic_fsub_ret_f64__offset(ptr addrspace(3) %ptr) nounwind
; GFX6-NEXT: v_mov_b32_e32 v4, v1
; GFX6-NEXT: v_mov_b32_e32 v3, v0
; GFX6-NEXT: v_add_f64 v[0:1], v[3:4], -4.0
-; GFX6-NEXT: ds_cmpst_rtn_b64 v[0:1], v2, v[3:4], v[0:1]
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 0xfff8, v2
+; GFX6-NEXT: ds_cmpst_rtn_b64 v[0:1], v5, v[3:4], v[0:1]
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[3:4]
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -1665,20 +1670,21 @@ define void @local_atomic_fsub_noret_f64__offset(ptr addrspace(3) %ptr) nounwind
; GFX6-LABEL: local_atomic_fsub_noret_f64__offset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfff8, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfff8, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b64 v[0:1], v2
+; GFX6-NEXT: ds_read_b64 v[1:2], v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_add_f64 v[3:4], v[0:1], -4.0
-; GFX6-NEXT: ds_cmpst_rtn_b64 v[3:4], v2, v[0:1], v[3:4]
+; GFX6-NEXT: v_add_f64 v[3:4], v[1:2], -4.0
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 0xfff8, v0
+; GFX6-NEXT: ds_cmpst_rtn_b64 v[3:4], v5, v[1:2], v[3:4]
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v0, v3
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[3:4], v[1:2]
+; GFX6-NEXT: v_mov_b32_e32 v1, v3
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
+; GFX6-NEXT: v_mov_b32_e32 v2, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB7_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3487,25 +3493,27 @@ define half @local_atomic_fsub_ret_f16__offset__align4(ptr addrspace(3) %ptr) no
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v0, v1
+; GFX6-NEXT: ds_read_b32 v1, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v2, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v2
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_add_f32_e32 v0, -4.0, v0
-; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX6-NEXT: ds_cmpst_rtn_b32 v0, v1, v2, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, v1
+; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v2
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 0xfffe, v0
+; GFX6-NEXT: v_add_f32_e32 v1, -4.0, v1
+; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v3, v2, v1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB12_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX6-NEXT: v_mov_b32_e32 v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(3) %ptr, i32 32767
%result = atomicrmw fsub ptr addrspace(3) %gep, half 4.0 seq_cst, align 4
@@ -3768,19 +3776,20 @@ define void @local_atomic_fsub_noret_f16__offset__align4(ptr addrspace(3) %ptr)
; GFX6-LABEL: local_atomic_fsub_noret_f16__offset__align4:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffe, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v1, v0
+; GFX6-NEXT: ds_read_b32 v1, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v1
; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffe, v0
; GFX6-NEXT: v_add_f32_e32 v2, -4.0, v2
; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
+; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v4, v1, v2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -5953,25 +5962,27 @@ define bfloat @local_atomic_fsub_ret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v0, v1
+; GFX6-NEXT: ds_read_b32 v1, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v2, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX6-NEXT: v_add_f32_e32 v0, -4.0, v0
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX6-NEXT: ds_cmpst_rtn_b32 v0, v1, v2, v0
+; GFX6-NEXT: v_mov_b32_e32 v2, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX6-NEXT: v_add_f32_e32 v3, -4.0, v3
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
+; GFX6-NEXT: v_or_b32_e32 v3, v4, v3
+; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v1, v2, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB18_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX6-NEXT: v_mov_b32_e32 v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(3) %ptr, i32 32767
%result = atomicrmw fsub ptr addrspace(3) %gep, bfloat 4.0 seq_cst, align 4
@@ -6310,9 +6321,9 @@ define void @local_atomic_fsub_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX6-LABEL: local_atomic_fsub_noret_bf16__offset__align4:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffe, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 0xfffe, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v1, v0
+; GFX6-NEXT: ds_read_b32 v1, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6321,8 +6332,9 @@ define void @local_atomic_fsub_noret_bf16__offset__align4(ptr addrspace(3) %ptr)
; GFX6-NEXT: v_add_f32_e32 v2, -4.0, v2
; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0xfffe, v0
; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
+; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v4, v1, v2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -6794,9 +6806,9 @@ define <2 x half> @local_atomic_fsub_ret_v2f16__offset(ptr addrspace(3) %ptr, <2
; GFX6-LABEL: local_atomic_fsub_ret_v2f16__offset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffc, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v3, v0
+; GFX6-NEXT: ds_read_b32 v3, v2
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
@@ -6814,9 +6826,10 @@ define <2 x half> @local_atomic_fsub_ret_v2f16__offset(ptr addrspace(3) %ptr, <2
; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
; GFX6-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX6-NEXT: v_add_i32_e32 v7, vcc, 0xfffc, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX6-NEXT: v_or_b32_e32 v3, v6, v3
-; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3
+; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v7, v4, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -7259,9 +7272,9 @@ define void @local_atomic_fsub_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
; GFX6-LABEL: local_atomic_fsub_noret_v2f16__offset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffc, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v3, v0
+; GFX6-NEXT: ds_read_b32 v3, v2
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
@@ -7279,9 +7292,10 @@ define void @local_atomic_fsub_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
; GFX6-NEXT: v_or_b32_e32 v4, v3, v4
+; GFX6-NEXT: v_add_i32_e32 v7, vcc, 0xfffc, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX6-NEXT: v_or_b32_e32 v3, v6, v3
-; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v0, v4, v3
+; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v7, v4, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -8225,9 +8239,9 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX6-LABEL: local_atomic_fsub_ret_v2bf16__offset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffc, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v2, v0
+; GFX6-NEXT: ds_read_b32 v2, v2
; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
@@ -8242,15 +8256,16 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX6-NEXT: v_sub_f32_e32 v7, v7, v2
+; GFX6-NEXT: v_sub_f32_e32 v6, v6, v1
; GFX6-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 0xfffc, v0
+; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
+; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v5, v3, v4
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
@@ -9151,9 +9166,9 @@ define void @local_atomic_fsub_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX6-LABEL: local_atomic_fsub_noret_v2bf16__ofset:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 0xfffc, v0
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0xfffc, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v2, v0
+; GFX6-NEXT: ds_read_b32 v2, v2
; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
@@ -9168,15 +9183,16 @@ define void @local_atomic_fsub_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX6-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_sub_f32_e32 v6, v6, v2
-; GFX6-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX6-NEXT: v_sub_f32_e32 v7, v7, v2
+; GFX6-NEXT: v_sub_f32_e32 v6, v6, v1
; GFX6-NEXT: v_alignbit_b32 v3, v3, v4, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v5, 16
-; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v7
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 0xfffc, v0
+; GFX6-NEXT: v_alignbit_b32 v4, v4, v6, 16
+; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v5, v3, v4
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
diff --git a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
index 13b1d76e14a00..e7eefafe31203 100644
--- a/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
+++ b/llvm/test/CodeGen/AMDGPU/ran-out-of-sgprs-allocation-failure.mir
@@ -44,10 +44,14 @@ body: |
; CHECK-NEXT: SI_SPILL_S32_SAVE $sgpr15, %stack.0, implicit $exec, implicit $sgpr32 :: (store (s32) into %stack.0, addrspace 5)
; CHECK-NEXT: SI_SPILL_S32_SAVE $sgpr14, %stack.1, implicit $exec, implicit $sgpr32 :: (store (s32) into %stack.1, addrspace 5)
; CHECK-NEXT: renamable $sgpr14_sgpr15 = V_CMP_GT_I32_e64 1, undef %18:vgpr_32, implicit $exec
- ; CHECK-NEXT: renamable $sgpr12_sgpr13 = V_CMP_NE_U32_e64 0, undef %18:vgpr_32, implicit $exec
+ ; CHECK-NEXT: renamable $sgpr12_sgpr13 = V_CMP_EQ_U32_e64 0, undef %18:vgpr_32, implicit $exec
; CHECK-NEXT: SI_SPILL_S64_SAVE killed renamable $sgpr12_sgpr13, %stack.3, implicit $exec, implicit $sgpr32 :: (store (s64) into %stack.3, align 4, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr12_sgpr13 = V_CMP_NE_U32_e64 0, undef %18:vgpr_32, implicit $exec
+ ; CHECK-NEXT: SI_SPILL_S64_SAVE killed renamable $sgpr12_sgpr13, %stack.4, implicit $exec, implicit $sgpr32 :: (store (s64) into %stack.4, align 4, addrspace 5)
; CHECK-NEXT: renamable $sgpr34_sgpr35 = V_CMP_GT_I32_e64 0, undef %18:vgpr_32, implicit $exec
; CHECK-NEXT: renamable $sgpr56 = S_MOV_B32 0
+ ; CHECK-NEXT: renamable $sgpr12_sgpr13 = V_CMP_EQ_U32_e64 undef $sgpr4, undef %18:vgpr_32, implicit $exec
+ ; CHECK-NEXT: SI_SPILL_S64_SAVE killed renamable $sgpr12_sgpr13, %stack.5, implicit $exec, implicit $sgpr32 :: (store (s64) into %stack.5, align 4, addrspace 5)
; CHECK-NEXT: [[COPY:%[0-9]+]]:vreg_1024_align2 = COPY renamable $sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63_sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71, implicit $exec
; CHECK-NEXT: renamable $sgpr100_sgpr101 = V_CMP_NE_U32_e64 1, undef %18:vgpr_32, implicit $exec
; CHECK-NEXT: renamable $sgpr57 = S_MOV_B32 1083786240
@@ -163,7 +167,7 @@ body: |
; CHECK-NEXT: successors: %bb.12(0x40000000), %bb.6(0x40000000)
; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr100_sgpr101
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $sgpr12_sgpr13 = SI_SPILL_S64_RESTORE %stack.3, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.3, align 4, addrspace 5)
+ ; CHECK-NEXT: renamable $sgpr12_sgpr13 = SI_SPILL_S64_RESTORE %stack.4, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.4, align 4, addrspace 5)
; CHECK-NEXT: renamable $sgpr12_sgpr13 = S_AND_B64 killed renamable $sgpr12_sgpr13, undef renamable $sgpr54_sgpr55, implicit-def dead $scc
; CHECK-NEXT: renamable $sgpr54_sgpr55 = V_CMP_GT_I32_e64 0, undef %18:vgpr_32, implicit $exec
; CHECK-NEXT: $exec = S_MOV_B64_term killed renamable $sgpr12_sgpr13
@@ -211,13 +215,13 @@ body: |
; CHECK-NEXT: $sgpr13 = SI_SPILL_S32_RESTORE %stack.0, implicit $exec, implicit $sgpr32 :: (load (s32) from %stack.0, addrspace 5)
; CHECK-NEXT: renamable $sgpr84 = COPY killed renamable $sgpr8
; CHECK-NEXT: renamable $sgpr33 = COPY killed renamable $sgpr16
- ; CHECK-NEXT: renamable $sgpr36_sgpr37 = COPY killed renamable $sgpr14_sgpr15
+ ; CHECK-NEXT: renamable $sgpr48_sgpr49 = COPY killed renamable $sgpr14_sgpr15
; CHECK-NEXT: dead $sgpr30_sgpr31 = SI_CALL undef renamable $sgpr12_sgpr13, 0, csr_amdgpu_gfx90ainsts, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13
; CHECK-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; CHECK-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; CHECK-NEXT: $sgpr8_sgpr9 = COPY renamable $sgpr82_sgpr83
; CHECK-NEXT: dead $sgpr30_sgpr31 = SI_CALL undef renamable $sgpr12_sgpr13, 0, csr_amdgpu_gfx90ainsts, implicit $sgpr8_sgpr9
- ; CHECK-NEXT: renamable $sgpr14_sgpr15 = COPY killed renamable $sgpr36_sgpr37
+ ; CHECK-NEXT: renamable $sgpr14_sgpr15 = COPY killed renamable $sgpr48_sgpr49
; CHECK-NEXT: renamable $sgpr16 = COPY killed renamable $sgpr33
; CHECK-NEXT: renamable $sgpr4_sgpr5 = COPY killed renamable $sgpr68_sgpr69
; CHECK-NEXT: renamable $sgpr6_sgpr7 = COPY killed renamable $sgpr70_sgpr71
@@ -253,7 +257,7 @@ body: |
; CHECK-NEXT: successors: %bb.15(0x40000000), %bb.14(0x40000000)
; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr100_sgpr101
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $sgpr12_sgpr13 = V_CMP_EQ_U32_e64 undef $sgpr4, undef %18:vgpr_32, implicit $exec
+ ; CHECK-NEXT: renamable $sgpr12_sgpr13 = SI_SPILL_S64_RESTORE %stack.5, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.5, align 4, addrspace 5)
; CHECK-NEXT: $vcc = S_AND_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def dead $scc
; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.15, implicit $vcc
; CHECK-NEXT: S_BRANCH %bb.14
@@ -266,7 +270,7 @@ body: |
; CHECK-NEXT: successors: %bb.11(0x40000000), %bb.16(0x40000000)
; CHECK-NEXT: liveins: $sgpr16, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x0000000000000003, $sgpr10_sgpr11, $sgpr14_sgpr15, $sgpr34_sgpr35, $sgpr100_sgpr101
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $sgpr12_sgpr13 = V_CMP_EQ_U32_e64 0, undef %18:vgpr_32, implicit $exec
+ ; CHECK-NEXT: renamable $sgpr12_sgpr13 = SI_SPILL_S64_RESTORE %stack.3, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.3, align 4, addrspace 5)
; CHECK-NEXT: $vcc = S_AND_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def dead $scc
; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.11, implicit $vcc
; CHECK-NEXT: {{ $}}
diff --git a/llvm/test/CodeGen/AMDGPU/structurize-hoist.ll b/llvm/test/CodeGen/AMDGPU/structurize-hoist.ll
index b4036517cc0d5..ade922e8af166 100644
--- a/llvm/test/CodeGen/AMDGPU/structurize-hoist.ll
+++ b/llvm/test/CodeGen/AMDGPU/structurize-hoist.ll
@@ -85,58 +85,57 @@ merge:
define amdgpu_kernel void @test_loop_with_if( ptr %ptr, i1 %cond) #0 {
; GFX900-LABEL: test_loop_with_if:
; GFX900: ; %bb.0: ; %entry
-; GFX900-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX900-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX900-NEXT: v_mov_b32_e32 v5, 0
-; GFX900-NEXT: s_mov_b64 s[4:5], 0
-; GFX900-NEXT: s_movk_i32 s10, 0xfe
+; GFX900-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX900-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX900-NEXT: v_mov_b32_e32 v6, 0
+; GFX900-NEXT: s_mov_b64 s[2:3], 0
+; GFX900-NEXT: s_movk_i32 s8, 0xfe
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-NEXT: s_bitcmp1_b32 s2, 0
-; GFX900-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX900-NEXT: v_cndmask_b32_e64 v3, 0, 1, s[2:3]
-; GFX900-NEXT: v_mov_b32_e32 v2, s1
-; GFX900-NEXT: s_xor_b64 s[2:3], s[2:3], -1
-; GFX900-NEXT: v_mov_b32_e32 v1, s0
-; GFX900-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v3
+; GFX900-NEXT: s_bitcmp1_b32 s0, 0
+; GFX900-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX900-NEXT: v_mov_b32_e32 v1, s6
+; GFX900-NEXT: v_cndmask_b32_e64 v5, 0, 1, s[0:1]
+; GFX900-NEXT: s_xor_b64 s[0:1], s[0:1], -1
+; GFX900-NEXT: v_mov_b32_e32 v2, s7
; GFX900-NEXT: s_branch .LBB2_2
; GFX900-NEXT: .LBB2_1: ; %latch
; GFX900-NEXT: ; in Loop: Header=BB2_2 Depth=1
-; GFX900-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX900-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_add_u32_e32 v5, 20, v3
-; GFX900-NEXT: v_cmp_lt_i32_e32 vcc, s10, v5
-; GFX900-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX900-NEXT: v_add_u32_e32 v6, 20, v3
+; GFX900-NEXT: v_cmp_lt_i32_e32 vcc, s8, v6
+; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX900-NEXT: flat_store_dword v[1:2], v3
-; GFX900-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX900-NEXT: s_cbranch_execz .LBB2_8
; GFX900-NEXT: .LBB2_2: ; %loop
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: flat_load_dwordx2 v[3:4], v[1:2]
-; GFX900-NEXT: s_and_b64 vcc, exec, s[0:1]
-; GFX900-NEXT: s_mov_b64 s[8:9], s[2:3]
-; GFX900-NEXT: s_mov_b64 s[6:7], 0
+; GFX900-NEXT: v_cmp_ne_u32_e32 vcc, 1, v5
+; GFX900-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX900-NEXT: s_mov_b64 s[4:5], 0
; GFX900-NEXT: s_cbranch_vccnz .LBB2_4
; GFX900-NEXT: ; %bb.3: ; %if
; GFX900-NEXT: ; in Loop: Header=BB2_2 Depth=1
-; GFX900-NEXT: v_cmp_gt_i32_e32 vcc, 11, v5
-; GFX900-NEXT: s_andn2_b64 s[8:9], s[2:3], exec
-; GFX900-NEXT: s_and_b64 s[12:13], vcc, exec
-; GFX900-NEXT: s_mov_b64 s[6:7], -1
-; GFX900-NEXT: s_or_b64 s[8:9], s[8:9], s[12:13]
+; GFX900-NEXT: v_cmp_gt_i32_e32 vcc, 11, v6
+; GFX900-NEXT: s_andn2_b64 s[6:7], s[0:1], exec
+; GFX900-NEXT: s_and_b64 s[10:11], vcc, exec
+; GFX900-NEXT: s_mov_b64 s[4:5], -1
+; GFX900-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
; GFX900-NEXT: .LBB2_4: ; %Flow
; GFX900-NEXT: ; in Loop: Header=BB2_2 Depth=1
-; GFX900-NEXT: s_and_saveexec_b64 s[12:13], s[8:9]
-; GFX900-NEXT: s_xor_b64 s[8:9], exec, s[12:13]
+; GFX900-NEXT: s_and_saveexec_b64 s[10:11], s[6:7]
+; GFX900-NEXT: s_xor_b64 s[6:7], exec, s[10:11]
; GFX900-NEXT: s_cbranch_execz .LBB2_6
; GFX900-NEXT: ; %bb.5: ; %else
; GFX900-NEXT: ; in Loop: Header=BB2_2 Depth=1
; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX900-NEXT: v_add_u32_e32 v3, v3, v4
-; GFX900-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
+; GFX900-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
; GFX900-NEXT: .LBB2_6: ; %Flow1
; GFX900-NEXT: ; in Loop: Header=BB2_2 Depth=1
-; GFX900-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX900-NEXT: s_and_saveexec_b64 s[8:9], s[6:7]
+; GFX900-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX900-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; GFX900-NEXT: s_cbranch_execz .LBB2_1
; GFX900-NEXT: ; %bb.7: ; %then
; GFX900-NEXT: ; in Loop: Header=BB2_2 Depth=1
diff --git a/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll b/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll
index 2221647f580cb..64f141908895d 100644
--- a/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll
+++ b/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll
@@ -32,54 +32,43 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-LABEL: kernel:
; GLOBALNESS1: ; %bb.0: ; %bb
; GLOBALNESS1-NEXT: s_mov_b64 s[36:37], s[6:7]
-; GLOBALNESS1-NEXT: s_load_dwordx4 s[52:55], s[8:9], 0x0
+; GLOBALNESS1-NEXT: s_load_dwordx4 s[68:71], s[8:9], 0x0
; GLOBALNESS1-NEXT: s_load_dword s6, s[8:9], 0x14
; GLOBALNESS1-NEXT: v_mov_b32_e32 v42, 0
; GLOBALNESS1-NEXT: v_pk_mov_b32 v[44:45], 0, 0
; GLOBALNESS1-NEXT: global_store_dword v[44:45], v42, off
; GLOBALNESS1-NEXT: s_waitcnt lgkmcnt(0)
-; GLOBALNESS1-NEXT: global_load_dword v2, v42, s[52:53]
+; GLOBALNESS1-NEXT: global_load_dword v2, v42, s[68:69]
; GLOBALNESS1-NEXT: s_mov_b64 s[38:39], s[4:5]
; GLOBALNESS1-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x18
; GLOBALNESS1-NEXT: s_load_dword s7, s[8:9], 0x20
; GLOBALNESS1-NEXT: s_add_u32 flat_scratch_lo, s12, s17
; GLOBALNESS1-NEXT: s_addc_u32 flat_scratch_hi, s13, 0
; GLOBALNESS1-NEXT: s_add_u32 s0, s0, s17
-; GLOBALNESS1-NEXT: s_addc_u32 s1, s1, 0
; GLOBALNESS1-NEXT: v_mov_b32_e32 v41, v0
; GLOBALNESS1-NEXT: v_mov_b32_e32 v0, 0
; GLOBALNESS1-NEXT: v_mov_b32_e32 v1, 0x40994400
-; GLOBALNESS1-NEXT: s_bitcmp1_b32 s54, 0
+; GLOBALNESS1-NEXT: s_addc_u32 s1, s1, 0
; GLOBALNESS1-NEXT: s_waitcnt lgkmcnt(0)
-; GLOBALNESS1-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
+; GLOBALNESS1-NEXT: v_cmp_ngt_f64_e64 s[50:51], s[4:5], v[0:1]
; GLOBALNESS1-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[4:5], 0
-; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5]
+; GLOBALNESS1-NEXT: ; implicit-def: $vgpr60 : SGPR spill to VGPR lane
+; GLOBALNESS1-NEXT: s_bitcmp1_b32 s70, 0
+; GLOBALNESS1-NEXT: v_writelane_b32 v60, s4, 0
+; GLOBALNESS1-NEXT: v_writelane_b32 v60, s5, 1
; GLOBALNESS1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v3, 0, 1, s[4:5]
-; GLOBALNESS1-NEXT: s_xor_b64 s[4:5], s[4:5], -1
-; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GLOBALNESS1-NEXT: s_xor_b64 s[52:53], s[4:5], -1
; GLOBALNESS1-NEXT: s_bitcmp1_b32 s6, 0
-; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[50:51], 1, v0
-; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v40, 0, 1, s[4:5]
; GLOBALNESS1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GLOBALNESS1-NEXT: s_xor_b64 s[4:5], s[4:5], -1
+; GLOBALNESS1-NEXT: s_xor_b64 s[64:65], s[4:5], -1
; GLOBALNESS1-NEXT: s_bitcmp1_b32 s7, 0
-; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[64:65], 1, v0
-; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
; GLOBALNESS1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GLOBALNESS1-NEXT: s_xor_b64 s[4:5], s[4:5], -1
+; GLOBALNESS1-NEXT: s_xor_b64 s[66:67], s[4:5], -1
+; GLOBALNESS1-NEXT: s_mov_b32 s84, s16
; GLOBALNESS1-NEXT: s_mov_b64 s[48:49], s[8:9]
-; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[8:9], 1, v1
-; GLOBALNESS1-NEXT: ; implicit-def: $vgpr57 : SGPR spill to VGPR lane
-; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[66:67], 1, v0
-; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GLOBALNESS1-NEXT: v_writelane_b32 v57, s8, 0
-; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[68:69], 1, v0
-; GLOBALNESS1-NEXT: v_writelane_b32 v57, s9, 1
-; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[70:71], 1, v3
-; GLOBALNESS1-NEXT: s_mov_b32 s82, s16
-; GLOBALNESS1-NEXT: s_mov_b32 s83, s15
-; GLOBALNESS1-NEXT: s_mov_b32 s84, s14
+; GLOBALNESS1-NEXT: s_mov_b32 s85, s15
+; GLOBALNESS1-NEXT: s_mov_b32 s86, s14
; GLOBALNESS1-NEXT: s_mov_b64 s[34:35], s[10:11]
; GLOBALNESS1-NEXT: v_mov_b32_e32 v46, 0x80
; GLOBALNESS1-NEXT: v_mov_b32_e32 v47, 0
@@ -87,32 +76,22 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: s_mov_b32 s32, 0
; GLOBALNESS1-NEXT: ; implicit-def: $vgpr58_vgpr59
; GLOBALNESS1-NEXT: s_waitcnt vmcnt(0)
-; GLOBALNESS1-NEXT: v_cmp_gt_i32_e32 vcc, 0, v2
-; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GLOBALNESS1-NEXT: v_cmp_gt_i32_e32 vcc, 1, v2
-; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GLOBALNESS1-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2
-; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[4:5], 1, v0
-; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GLOBALNESS1-NEXT: v_writelane_b32 v57, s4, 2
-; GLOBALNESS1-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
-; GLOBALNESS1-NEXT: v_writelane_b32 v57, s5, 3
-; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[4:5], 1, v3
-; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GLOBALNESS1-NEXT: v_writelane_b32 v57, s4, 4
-; GLOBALNESS1-NEXT: v_writelane_b32 v57, s5, 5
-; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[4:5], 1, v2
-; GLOBALNESS1-NEXT: v_writelane_b32 v57, s4, 6
-; GLOBALNESS1-NEXT: v_writelane_b32 v57, s5, 7
-; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[80:81], 1, v1
-; GLOBALNESS1-NEXT: v_writelane_b32 v57, s70, 8
-; GLOBALNESS1-NEXT: v_writelane_b32 v57, s71, 9
+; GLOBALNESS1-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v2
+; GLOBALNESS1-NEXT: v_writelane_b32 v60, s4, 2
+; GLOBALNESS1-NEXT: v_writelane_b32 v60, s5, 3
+; GLOBALNESS1-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v2
+; GLOBALNESS1-NEXT: v_writelane_b32 v60, s4, 4
+; GLOBALNESS1-NEXT: v_writelane_b32 v60, s5, 5
+; GLOBALNESS1-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v2
+; GLOBALNESS1-NEXT: v_cmp_gt_i32_e64 s[54:55], 1, v2
+; GLOBALNESS1-NEXT: v_writelane_b32 v60, s4, 6
+; GLOBALNESS1-NEXT: v_writelane_b32 v60, s5, 7
; GLOBALNESS1-NEXT: s_branch .LBB1_4
; GLOBALNESS1-NEXT: .LBB1_1: ; %bb70.i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT: v_readlane_b32 s6, v57, 6
-; GLOBALNESS1-NEXT: v_readlane_b32 s7, v57, 7
-; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[6:7]
+; GLOBALNESS1-NEXT: v_readlane_b32 s6, v60, 6
+; GLOBALNESS1-NEXT: v_readlane_b32 s7, v60, 7
+; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
; GLOBALNESS1-NEXT: s_cbranch_vccz .LBB1_28
; GLOBALNESS1-NEXT: .LBB1_2: ; %Flow15
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
@@ -126,10 +105,10 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: .LBB1_4: ; %bb5
; GLOBALNESS1-NEXT: ; =>This Loop Header: Depth=1
; GLOBALNESS1-NEXT: ; Child Loop BB1_16 Depth 2
-; GLOBALNESS1-NEXT: flat_load_dword v40, v[46:47]
+; GLOBALNESS1-NEXT: flat_load_dword v56, v[46:47]
; GLOBALNESS1-NEXT: s_add_u32 s8, s48, 40
; GLOBALNESS1-NEXT: buffer_store_dword v42, off, s[0:3], 0
-; GLOBALNESS1-NEXT: flat_load_dword v56, v[46:47]
+; GLOBALNESS1-NEXT: flat_load_dword v57, v[46:47]
; GLOBALNESS1-NEXT: s_addc_u32 s9, s49, 0
; GLOBALNESS1-NEXT: s_getpc_b64 s[4:5]
; GLOBALNESS1-NEXT: s_add_u32 s4, s4, wobble at gotpcrel32@lo+4
@@ -138,14 +117,14 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], s[38:39]
; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], s[36:37]
; GLOBALNESS1-NEXT: s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS1-NEXT: s_mov_b32 s12, s84
-; GLOBALNESS1-NEXT: s_mov_b32 s13, s83
-; GLOBALNESS1-NEXT: s_mov_b32 s14, s82
+; GLOBALNESS1-NEXT: s_mov_b32 s12, s86
+; GLOBALNESS1-NEXT: s_mov_b32 s13, s85
+; GLOBALNESS1-NEXT: s_mov_b32 s14, s84
; GLOBALNESS1-NEXT: v_mov_b32_e32 v31, v41
; GLOBALNESS1-NEXT: ; implicit-def: $sgpr15
; GLOBALNESS1-NEXT: s_waitcnt lgkmcnt(0)
; GLOBALNESS1-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[70:71]
+; GLOBALNESS1-NEXT: v_cmp_ne_u32_e32 vcc, 1, v40
; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], -1
; GLOBALNESS1-NEXT: ; implicit-def: $sgpr8_sgpr9
; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_9
@@ -153,12 +132,12 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS1-NEXT: s_mov_b64 s[8:9], -1
; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], 0
-; GLOBALNESS1-NEXT: s_cmp_lt_i32 s55, 1
+; GLOBALNESS1-NEXT: s_cmp_lt_i32 s71, 1
; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], -1
; GLOBALNESS1-NEXT: s_cbranch_scc1 .LBB1_7
; GLOBALNESS1-NEXT: ; %bb.6: ; %LeafBlock12
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT: s_cmp_lg_u32 s55, 1
+; GLOBALNESS1-NEXT: s_cmp_lg_u32 s71, 1
; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], 0
; GLOBALNESS1-NEXT: s_cselect_b64 s[4:5], -1, 0
; GLOBALNESS1-NEXT: .LBB1_7: ; %Flow26
@@ -167,7 +146,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_9
; GLOBALNESS1-NEXT: ; %bb.8: ; %LeafBlock
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT: s_cmp_lg_u32 s55, 0
+; GLOBALNESS1-NEXT: s_cmp_lg_u32 s71, 0
; GLOBALNESS1-NEXT: s_mov_b64 s[8:9], 0
; GLOBALNESS1-NEXT: s_cselect_b64 s[4:5], -1, 0
; GLOBALNESS1-NEXT: .LBB1_9: ; %Flow25
@@ -181,95 +160,94 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: flat_load_dword v0, v[44:45]
; GLOBALNESS1-NEXT: v_mov_b32_e32 v1, 0x3ff00000
; GLOBALNESS1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GLOBALNESS1-NEXT: v_cmp_gt_i32_e64 s[86:87], 0, v0
+; GLOBALNESS1-NEXT: v_cmp_gt_i32_e64 s[68:69], 0, v0
; GLOBALNESS1-NEXT: v_mov_b32_e32 v0, 0
-; GLOBALNESS1-NEXT: s_and_saveexec_b64 s[52:53], s[86:87]
+; GLOBALNESS1-NEXT: s_and_saveexec_b64 s[98:99], s[68:69]
; GLOBALNESS1-NEXT: s_cbranch_execz .LBB1_25
; GLOBALNESS1-NEXT: ; %bb.11: ; %bb33.i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS1-NEXT: global_load_dwordx2 v[0:1], v[44:45], off
-; GLOBALNESS1-NEXT: v_writelane_b32 v57, s8, 10
-; GLOBALNESS1-NEXT: v_writelane_b32 v57, s9, 11
-; GLOBALNESS1-NEXT: v_readlane_b32 s4, v57, 2
-; GLOBALNESS1-NEXT: v_readlane_b32 s5, v57, 3
-; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[4:5]
+; GLOBALNESS1-NEXT: v_writelane_b32 v60, s8, 8
+; GLOBALNESS1-NEXT: v_writelane_b32 v60, s9, 9
+; GLOBALNESS1-NEXT: v_readlane_b32 s4, v60, 2
+; GLOBALNESS1-NEXT: v_readlane_b32 s5, v60, 3
+; GLOBALNESS1-NEXT: s_mov_b32 s87, s71
+; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_13
; GLOBALNESS1-NEXT: ; %bb.12: ; %bb39.i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS1-NEXT: global_store_dwordx2 v[44:45], v[42:43], off
; GLOBALNESS1-NEXT: .LBB1_13: ; %bb44.lr.ph.i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT: v_cmp_ne_u32_e32 vcc, 0, v56
-; GLOBALNESS1-NEXT: v_cndmask_b32_e32 v2, 0, v40, vcc
+; GLOBALNESS1-NEXT: v_cmp_ne_u32_e32 vcc, 0, v57
+; GLOBALNESS1-NEXT: v_cndmask_b32_e32 v2, 0, v56, vcc
+; GLOBALNESS1-NEXT: v_cmp_eq_u32_e64 s[70:71], 0, v2
; GLOBALNESS1-NEXT: s_waitcnt vmcnt(0)
-; GLOBALNESS1-NEXT: v_cmp_nlt_f64_e32 vcc, 0, v[0:1]
-; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GLOBALNESS1-NEXT: v_cmp_eq_u32_e64 s[96:97], 0, v2
-; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[98:99], 1, v0
+; GLOBALNESS1-NEXT: v_cmp_nlt_f64_e64 s[80:81], 0, v[0:1]
; GLOBALNESS1-NEXT: s_branch .LBB1_16
; GLOBALNESS1-NEXT: .LBB1_14: ; %Flow16
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_16 Depth=2
; GLOBALNESS1-NEXT: s_or_b64 exec, exec, s[4:5]
; GLOBALNESS1-NEXT: .LBB1_15: ; %bb63.i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[68:69]
+; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[66:67]
; GLOBALNESS1-NEXT: s_cbranch_vccz .LBB1_24
; GLOBALNESS1-NEXT: .LBB1_16: ; %bb44.i
; GLOBALNESS1-NEXT: ; Parent Loop BB1_4 Depth=1
; GLOBALNESS1-NEXT: ; => This Inner Loop Header: Depth=2
-; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[64:65]
+; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[52:53]
; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_15
; GLOBALNESS1-NEXT: ; %bb.17: ; %bb46.i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[66:67]
+; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[64:65]
; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_15
; GLOBALNESS1-NEXT: ; %bb.18: ; %bb50.i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[50:51]
+; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[50:51]
; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_21
; GLOBALNESS1-NEXT: ; %bb.19: ; %bb3.i.i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS1-NEXT: v_readlane_b32 s4, v57, 0
-; GLOBALNESS1-NEXT: v_readlane_b32 s5, v57, 1
-; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[4:5]
+; GLOBALNESS1-NEXT: v_readlane_b32 s4, v60, 0
+; GLOBALNESS1-NEXT: v_readlane_b32 s5, v60, 1
+; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_21
; GLOBALNESS1-NEXT: ; %bb.20: ; %bb6.i.i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[98:99]
+; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[80:81]
; GLOBALNESS1-NEXT: .LBB1_21: ; %spam.exit.i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[80:81]
+; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[54:55]
; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_15
; GLOBALNESS1-NEXT: ; %bb.22: ; %bb55.i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS1-NEXT: s_add_u32 s70, s48, 40
-; GLOBALNESS1-NEXT: s_addc_u32 s71, s49, 0
+; GLOBALNESS1-NEXT: s_add_u32 s82, s48, 40
+; GLOBALNESS1-NEXT: s_addc_u32 s83, s49, 0
; GLOBALNESS1-NEXT: s_getpc_b64 s[4:5]
; GLOBALNESS1-NEXT: s_add_u32 s4, s4, wobble at gotpcrel32@lo+4
; GLOBALNESS1-NEXT: s_addc_u32 s5, s5, wobble at gotpcrel32@hi+12
-; GLOBALNESS1-NEXT: s_load_dwordx2 s[54:55], s[4:5], 0x0
+; GLOBALNESS1-NEXT: s_load_dwordx2 s[96:97], s[4:5], 0x0
; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], s[38:39]
; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], s[36:37]
-; GLOBALNESS1-NEXT: s_mov_b64 s[8:9], s[70:71]
+; GLOBALNESS1-NEXT: s_mov_b64 s[8:9], s[82:83]
; GLOBALNESS1-NEXT: s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS1-NEXT: s_mov_b32 s12, s84
-; GLOBALNESS1-NEXT: s_mov_b32 s13, s83
-; GLOBALNESS1-NEXT: s_mov_b32 s14, s82
+; GLOBALNESS1-NEXT: s_mov_b32 s12, s86
+; GLOBALNESS1-NEXT: s_mov_b32 s13, s85
+; GLOBALNESS1-NEXT: s_mov_b32 s14, s84
; GLOBALNESS1-NEXT: v_mov_b32_e32 v31, v41
; GLOBALNESS1-NEXT: s_waitcnt lgkmcnt(0)
-; GLOBALNESS1-NEXT: s_swappc_b64 s[30:31], s[54:55]
+; GLOBALNESS1-NEXT: s_swappc_b64 s[30:31], s[96:97]
; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], s[38:39]
; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], s[36:37]
-; GLOBALNESS1-NEXT: s_mov_b64 s[8:9], s[70:71]
+; GLOBALNESS1-NEXT: s_mov_b64 s[8:9], s[82:83]
; GLOBALNESS1-NEXT: s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS1-NEXT: s_mov_b32 s12, s84
-; GLOBALNESS1-NEXT: s_mov_b32 s13, s83
-; GLOBALNESS1-NEXT: s_mov_b32 s14, s82
+; GLOBALNESS1-NEXT: s_mov_b32 s12, s86
+; GLOBALNESS1-NEXT: s_mov_b32 s13, s85
+; GLOBALNESS1-NEXT: s_mov_b32 s14, s84
; GLOBALNESS1-NEXT: v_mov_b32_e32 v31, v41
; GLOBALNESS1-NEXT: global_store_dwordx2 v[44:45], v[58:59], off
; GLOBALNESS1-NEXT: ; implicit-def: $sgpr15
-; GLOBALNESS1-NEXT: s_swappc_b64 s[30:31], s[54:55]
-; GLOBALNESS1-NEXT: s_and_saveexec_b64 s[4:5], s[96:97]
+; GLOBALNESS1-NEXT: s_swappc_b64 s[30:31], s[96:97]
+; GLOBALNESS1-NEXT: s_and_saveexec_b64 s[4:5], s[70:71]
; GLOBALNESS1-NEXT: s_cbranch_execz .LBB1_14
; GLOBALNESS1-NEXT: ; %bb.23: ; %bb62.i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_16 Depth=2
@@ -277,24 +255,20 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: s_branch .LBB1_14
; GLOBALNESS1-NEXT: .LBB1_24: ; %Flow23
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT: s_load_dwordx4 s[4:7], s[48:49], 0x0
-; GLOBALNESS1-NEXT: v_readlane_b32 s70, v57, 8
-; GLOBALNESS1-NEXT: v_readlane_b32 s8, v57, 10
+; GLOBALNESS1-NEXT: v_readlane_b32 s8, v60, 8
; GLOBALNESS1-NEXT: v_pk_mov_b32 v[0:1], 0, 0
-; GLOBALNESS1-NEXT: v_readlane_b32 s71, v57, 9
-; GLOBALNESS1-NEXT: s_waitcnt lgkmcnt(0)
-; GLOBALNESS1-NEXT: s_mov_b32 s55, s7
-; GLOBALNESS1-NEXT: v_readlane_b32 s9, v57, 11
+; GLOBALNESS1-NEXT: s_mov_b32 s71, s87
+; GLOBALNESS1-NEXT: v_readlane_b32 s9, v60, 9
; GLOBALNESS1-NEXT: .LBB1_25: ; %Flow24
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT: s_or_b64 exec, exec, s[52:53]
-; GLOBALNESS1-NEXT: s_and_saveexec_b64 s[4:5], s[86:87]
+; GLOBALNESS1-NEXT: s_or_b64 exec, exec, s[98:99]
+; GLOBALNESS1-NEXT: s_and_saveexec_b64 s[4:5], s[68:69]
; GLOBALNESS1-NEXT: s_cbranch_execz .LBB1_2
; GLOBALNESS1-NEXT: ; %bb.26: ; %bb67.i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS1-NEXT: v_readlane_b32 s6, v57, 4
-; GLOBALNESS1-NEXT: v_readlane_b32 s7, v57, 5
-; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[6:7]
+; GLOBALNESS1-NEXT: v_readlane_b32 s6, v60, 4
+; GLOBALNESS1-NEXT: v_readlane_b32 s7, v60, 5
+; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_1
; GLOBALNESS1-NEXT: ; %bb.27: ; %bb69.i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
@@ -317,9 +291,9 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], s[38:39]
; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], s[36:37]
; GLOBALNESS1-NEXT: s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS1-NEXT: s_mov_b32 s12, s84
-; GLOBALNESS1-NEXT: s_mov_b32 s13, s83
-; GLOBALNESS1-NEXT: s_mov_b32 s14, s82
+; GLOBALNESS1-NEXT: s_mov_b32 s12, s86
+; GLOBALNESS1-NEXT: s_mov_b32 s13, s85
+; GLOBALNESS1-NEXT: s_mov_b32 s14, s84
; GLOBALNESS1-NEXT: v_mov_b32_e32 v31, v41
; GLOBALNESS1-NEXT: ; implicit-def: $sgpr15
; GLOBALNESS1-NEXT: s_swappc_b64 s[30:31], s[16:17]
@@ -336,9 +310,9 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], s[38:39]
; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], s[36:37]
; GLOBALNESS1-NEXT: s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS1-NEXT: s_mov_b32 s12, s84
-; GLOBALNESS1-NEXT: s_mov_b32 s13, s83
-; GLOBALNESS1-NEXT: s_mov_b32 s14, s82
+; GLOBALNESS1-NEXT: s_mov_b32 s12, s86
+; GLOBALNESS1-NEXT: s_mov_b32 s13, s85
+; GLOBALNESS1-NEXT: s_mov_b32 s14, s84
; GLOBALNESS1-NEXT: v_mov_b32_e32 v31, v41
; GLOBALNESS1-NEXT: ; implicit-def: $sgpr15
; GLOBALNESS1-NEXT: s_swappc_b64 s[30:31], s[16:17]
@@ -347,54 +321,43 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-LABEL: kernel:
; GLOBALNESS0: ; %bb.0: ; %bb
; GLOBALNESS0-NEXT: s_mov_b64 s[36:37], s[6:7]
-; GLOBALNESS0-NEXT: s_load_dwordx4 s[52:55], s[8:9], 0x0
+; GLOBALNESS0-NEXT: s_load_dwordx4 s[68:71], s[8:9], 0x0
; GLOBALNESS0-NEXT: s_load_dword s6, s[8:9], 0x14
; GLOBALNESS0-NEXT: v_mov_b32_e32 v42, 0
; GLOBALNESS0-NEXT: v_pk_mov_b32 v[44:45], 0, 0
; GLOBALNESS0-NEXT: global_store_dword v[44:45], v42, off
; GLOBALNESS0-NEXT: s_waitcnt lgkmcnt(0)
-; GLOBALNESS0-NEXT: global_load_dword v2, v42, s[52:53]
+; GLOBALNESS0-NEXT: global_load_dword v2, v42, s[68:69]
; GLOBALNESS0-NEXT: s_mov_b64 s[38:39], s[4:5]
; GLOBALNESS0-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x18
; GLOBALNESS0-NEXT: s_load_dword s7, s[8:9], 0x20
; GLOBALNESS0-NEXT: s_add_u32 flat_scratch_lo, s12, s17
; GLOBALNESS0-NEXT: s_addc_u32 flat_scratch_hi, s13, 0
; GLOBALNESS0-NEXT: s_add_u32 s0, s0, s17
-; GLOBALNESS0-NEXT: s_addc_u32 s1, s1, 0
; GLOBALNESS0-NEXT: v_mov_b32_e32 v41, v0
; GLOBALNESS0-NEXT: v_mov_b32_e32 v0, 0
; GLOBALNESS0-NEXT: v_mov_b32_e32 v1, 0x40994400
-; GLOBALNESS0-NEXT: s_bitcmp1_b32 s54, 0
+; GLOBALNESS0-NEXT: s_addc_u32 s1, s1, 0
; GLOBALNESS0-NEXT: s_waitcnt lgkmcnt(0)
-; GLOBALNESS0-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[0:1]
+; GLOBALNESS0-NEXT: v_cmp_ngt_f64_e64 s[50:51], s[4:5], v[0:1]
; GLOBALNESS0-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[4:5], 0
-; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5]
+; GLOBALNESS0-NEXT: ; implicit-def: $vgpr60 : SGPR spill to VGPR lane
+; GLOBALNESS0-NEXT: s_bitcmp1_b32 s70, 0
+; GLOBALNESS0-NEXT: v_writelane_b32 v60, s4, 0
+; GLOBALNESS0-NEXT: v_writelane_b32 v60, s5, 1
; GLOBALNESS0-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v3, 0, 1, s[4:5]
-; GLOBALNESS0-NEXT: s_xor_b64 s[4:5], s[4:5], -1
-; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
+; GLOBALNESS0-NEXT: s_xor_b64 s[52:53], s[4:5], -1
; GLOBALNESS0-NEXT: s_bitcmp1_b32 s6, 0
-; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[50:51], 1, v0
-; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v40, 0, 1, s[4:5]
; GLOBALNESS0-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GLOBALNESS0-NEXT: s_xor_b64 s[4:5], s[4:5], -1
+; GLOBALNESS0-NEXT: s_xor_b64 s[64:65], s[4:5], -1
; GLOBALNESS0-NEXT: s_bitcmp1_b32 s7, 0
-; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[64:65], 1, v0
-; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
; GLOBALNESS0-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GLOBALNESS0-NEXT: s_xor_b64 s[4:5], s[4:5], -1
+; GLOBALNESS0-NEXT: s_xor_b64 s[66:67], s[4:5], -1
+; GLOBALNESS0-NEXT: s_mov_b32 s82, s16
; GLOBALNESS0-NEXT: s_mov_b64 s[48:49], s[8:9]
-; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[8:9], 1, v1
-; GLOBALNESS0-NEXT: ; implicit-def: $vgpr57 : SGPR spill to VGPR lane
-; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[66:67], 1, v0
-; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GLOBALNESS0-NEXT: v_writelane_b32 v57, s8, 0
-; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[68:69], 1, v0
-; GLOBALNESS0-NEXT: v_writelane_b32 v57, s9, 1
-; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[84:85], 1, v3
-; GLOBALNESS0-NEXT: s_mov_b32 s70, s16
-; GLOBALNESS0-NEXT: s_mov_b32 s71, s15
-; GLOBALNESS0-NEXT: s_mov_b32 s82, s14
+; GLOBALNESS0-NEXT: s_mov_b32 s83, s15
+; GLOBALNESS0-NEXT: s_mov_b32 s84, s14
; GLOBALNESS0-NEXT: s_mov_b64 s[34:35], s[10:11]
; GLOBALNESS0-NEXT: v_mov_b32_e32 v46, 0x80
; GLOBALNESS0-NEXT: v_mov_b32_e32 v47, 0
@@ -402,32 +365,22 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: s_mov_b32 s32, 0
; GLOBALNESS0-NEXT: ; implicit-def: $vgpr58_vgpr59
; GLOBALNESS0-NEXT: s_waitcnt vmcnt(0)
-; GLOBALNESS0-NEXT: v_cmp_gt_i32_e32 vcc, 0, v2
-; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GLOBALNESS0-NEXT: v_cmp_gt_i32_e32 vcc, 1, v2
-; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GLOBALNESS0-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2
-; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[4:5], 1, v0
-; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GLOBALNESS0-NEXT: v_writelane_b32 v57, s4, 2
-; GLOBALNESS0-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
-; GLOBALNESS0-NEXT: v_writelane_b32 v57, s5, 3
-; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[4:5], 1, v3
-; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GLOBALNESS0-NEXT: v_writelane_b32 v57, s4, 4
-; GLOBALNESS0-NEXT: v_writelane_b32 v57, s5, 5
-; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[4:5], 1, v2
-; GLOBALNESS0-NEXT: v_writelane_b32 v57, s4, 6
-; GLOBALNESS0-NEXT: v_writelane_b32 v57, s5, 7
-; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[80:81], 1, v1
-; GLOBALNESS0-NEXT: v_writelane_b32 v57, s84, 8
-; GLOBALNESS0-NEXT: v_writelane_b32 v57, s85, 9
+; GLOBALNESS0-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v2
+; GLOBALNESS0-NEXT: v_writelane_b32 v60, s4, 2
+; GLOBALNESS0-NEXT: v_writelane_b32 v60, s5, 3
+; GLOBALNESS0-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v2
+; GLOBALNESS0-NEXT: v_writelane_b32 v60, s4, 4
+; GLOBALNESS0-NEXT: v_writelane_b32 v60, s5, 5
+; GLOBALNESS0-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v2
+; GLOBALNESS0-NEXT: v_cmp_gt_i32_e64 s[54:55], 1, v2
+; GLOBALNESS0-NEXT: v_writelane_b32 v60, s4, 6
+; GLOBALNESS0-NEXT: v_writelane_b32 v60, s5, 7
; GLOBALNESS0-NEXT: s_branch .LBB1_4
; GLOBALNESS0-NEXT: .LBB1_1: ; %bb70.i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT: v_readlane_b32 s6, v57, 6
-; GLOBALNESS0-NEXT: v_readlane_b32 s7, v57, 7
-; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[6:7]
+; GLOBALNESS0-NEXT: v_readlane_b32 s6, v60, 6
+; GLOBALNESS0-NEXT: v_readlane_b32 s7, v60, 7
+; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[6:7]
; GLOBALNESS0-NEXT: s_cbranch_vccz .LBB1_28
; GLOBALNESS0-NEXT: .LBB1_2: ; %Flow15
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
@@ -441,10 +394,10 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: .LBB1_4: ; %bb5
; GLOBALNESS0-NEXT: ; =>This Loop Header: Depth=1
; GLOBALNESS0-NEXT: ; Child Loop BB1_16 Depth 2
-; GLOBALNESS0-NEXT: flat_load_dword v40, v[46:47]
+; GLOBALNESS0-NEXT: flat_load_dword v56, v[46:47]
; GLOBALNESS0-NEXT: s_add_u32 s8, s48, 40
; GLOBALNESS0-NEXT: buffer_store_dword v42, off, s[0:3], 0
-; GLOBALNESS0-NEXT: flat_load_dword v56, v[46:47]
+; GLOBALNESS0-NEXT: flat_load_dword v57, v[46:47]
; GLOBALNESS0-NEXT: s_addc_u32 s9, s49, 0
; GLOBALNESS0-NEXT: s_getpc_b64 s[4:5]
; GLOBALNESS0-NEXT: s_add_u32 s4, s4, wobble at gotpcrel32@lo+4
@@ -453,14 +406,14 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], s[38:39]
; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], s[36:37]
; GLOBALNESS0-NEXT: s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS0-NEXT: s_mov_b32 s12, s82
-; GLOBALNESS0-NEXT: s_mov_b32 s13, s71
-; GLOBALNESS0-NEXT: s_mov_b32 s14, s70
+; GLOBALNESS0-NEXT: s_mov_b32 s12, s84
+; GLOBALNESS0-NEXT: s_mov_b32 s13, s83
+; GLOBALNESS0-NEXT: s_mov_b32 s14, s82
; GLOBALNESS0-NEXT: v_mov_b32_e32 v31, v41
; GLOBALNESS0-NEXT: ; implicit-def: $sgpr15
; GLOBALNESS0-NEXT: s_waitcnt lgkmcnt(0)
; GLOBALNESS0-NEXT: s_swappc_b64 s[30:31], s[16:17]
-; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[84:85]
+; GLOBALNESS0-NEXT: v_cmp_ne_u32_e32 vcc, 1, v40
; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], -1
; GLOBALNESS0-NEXT: ; implicit-def: $sgpr8_sgpr9
; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_9
@@ -468,12 +421,12 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS0-NEXT: s_mov_b64 s[8:9], -1
; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], 0
-; GLOBALNESS0-NEXT: s_cmp_lt_i32 s55, 1
+; GLOBALNESS0-NEXT: s_cmp_lt_i32 s71, 1
; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], -1
; GLOBALNESS0-NEXT: s_cbranch_scc1 .LBB1_7
; GLOBALNESS0-NEXT: ; %bb.6: ; %LeafBlock12
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT: s_cmp_lg_u32 s55, 1
+; GLOBALNESS0-NEXT: s_cmp_lg_u32 s71, 1
; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], 0
; GLOBALNESS0-NEXT: s_cselect_b64 s[4:5], -1, 0
; GLOBALNESS0-NEXT: .LBB1_7: ; %Flow26
@@ -482,7 +435,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_9
; GLOBALNESS0-NEXT: ; %bb.8: ; %LeafBlock
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT: s_cmp_lg_u32 s55, 0
+; GLOBALNESS0-NEXT: s_cmp_lg_u32 s71, 0
; GLOBALNESS0-NEXT: s_mov_b64 s[8:9], 0
; GLOBALNESS0-NEXT: s_cselect_b64 s[4:5], -1, 0
; GLOBALNESS0-NEXT: .LBB1_9: ; %Flow25
@@ -496,96 +449,93 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: flat_load_dword v0, v[44:45]
; GLOBALNESS0-NEXT: v_mov_b32_e32 v1, 0x3ff00000
; GLOBALNESS0-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GLOBALNESS0-NEXT: v_cmp_gt_i32_e64 s[86:87], 0, v0
+; GLOBALNESS0-NEXT: v_cmp_gt_i32_e64 s[68:69], 0, v0
; GLOBALNESS0-NEXT: v_mov_b32_e32 v0, 0
-; GLOBALNESS0-NEXT: s_and_saveexec_b64 s[52:53], s[86:87]
+; GLOBALNESS0-NEXT: s_and_saveexec_b64 s[98:99], s[68:69]
; GLOBALNESS0-NEXT: s_cbranch_execz .LBB1_25
; GLOBALNESS0-NEXT: ; %bb.11: ; %bb33.i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS0-NEXT: global_load_dwordx2 v[0:1], v[44:45], off
-; GLOBALNESS0-NEXT: v_writelane_b32 v57, s8, 10
-; GLOBALNESS0-NEXT: v_writelane_b32 v57, s9, 11
-; GLOBALNESS0-NEXT: v_readlane_b32 s4, v57, 2
-; GLOBALNESS0-NEXT: v_readlane_b32 s5, v57, 3
-; GLOBALNESS0-NEXT: s_mov_b32 s83, s55
-; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[4:5]
+; GLOBALNESS0-NEXT: v_writelane_b32 v60, s8, 8
+; GLOBALNESS0-NEXT: v_writelane_b32 v60, s9, 9
+; GLOBALNESS0-NEXT: v_readlane_b32 s4, v60, 2
+; GLOBALNESS0-NEXT: v_readlane_b32 s5, v60, 3
+; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_13
; GLOBALNESS0-NEXT: ; %bb.12: ; %bb39.i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS0-NEXT: global_store_dwordx2 v[44:45], v[42:43], off
; GLOBALNESS0-NEXT: .LBB1_13: ; %bb44.lr.ph.i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT: v_cmp_ne_u32_e32 vcc, 0, v56
-; GLOBALNESS0-NEXT: v_cndmask_b32_e32 v2, 0, v40, vcc
+; GLOBALNESS0-NEXT: v_cmp_ne_u32_e32 vcc, 0, v57
+; GLOBALNESS0-NEXT: v_cndmask_b32_e32 v2, 0, v56, vcc
+; GLOBALNESS0-NEXT: v_cmp_eq_u32_e64 s[70:71], 0, v2
; GLOBALNESS0-NEXT: s_waitcnt vmcnt(0)
-; GLOBALNESS0-NEXT: v_cmp_nlt_f64_e32 vcc, 0, v[0:1]
-; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GLOBALNESS0-NEXT: v_cmp_eq_u32_e64 s[96:97], 0, v2
-; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[98:99], 1, v0
+; GLOBALNESS0-NEXT: v_cmp_nlt_f64_e64 s[80:81], 0, v[0:1]
; GLOBALNESS0-NEXT: s_branch .LBB1_16
; GLOBALNESS0-NEXT: .LBB1_14: ; %Flow16
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_16 Depth=2
; GLOBALNESS0-NEXT: s_or_b64 exec, exec, s[4:5]
; GLOBALNESS0-NEXT: .LBB1_15: ; %bb63.i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[68:69]
+; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[66:67]
; GLOBALNESS0-NEXT: s_cbranch_vccz .LBB1_24
; GLOBALNESS0-NEXT: .LBB1_16: ; %bb44.i
; GLOBALNESS0-NEXT: ; Parent Loop BB1_4 Depth=1
; GLOBALNESS0-NEXT: ; => This Inner Loop Header: Depth=2
-; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[64:65]
+; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[52:53]
; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_15
; GLOBALNESS0-NEXT: ; %bb.17: ; %bb46.i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[66:67]
+; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[64:65]
; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_15
; GLOBALNESS0-NEXT: ; %bb.18: ; %bb50.i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[50:51]
+; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[50:51]
; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_21
; GLOBALNESS0-NEXT: ; %bb.19: ; %bb3.i.i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS0-NEXT: v_readlane_b32 s4, v57, 0
-; GLOBALNESS0-NEXT: v_readlane_b32 s5, v57, 1
-; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[4:5]
+; GLOBALNESS0-NEXT: v_readlane_b32 s4, v60, 0
+; GLOBALNESS0-NEXT: v_readlane_b32 s5, v60, 1
+; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[4:5]
; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_21
; GLOBALNESS0-NEXT: ; %bb.20: ; %bb6.i.i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[98:99]
+; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[80:81]
; GLOBALNESS0-NEXT: .LBB1_21: ; %spam.exit.i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[80:81]
+; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[54:55]
; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_15
; GLOBALNESS0-NEXT: ; %bb.22: ; %bb55.i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_16 Depth=2
-; GLOBALNESS0-NEXT: s_add_u32 s84, s48, 40
-; GLOBALNESS0-NEXT: s_addc_u32 s85, s49, 0
+; GLOBALNESS0-NEXT: s_add_u32 s86, s48, 40
+; GLOBALNESS0-NEXT: s_addc_u32 s87, s49, 0
; GLOBALNESS0-NEXT: s_getpc_b64 s[4:5]
; GLOBALNESS0-NEXT: s_add_u32 s4, s4, wobble at gotpcrel32@lo+4
; GLOBALNESS0-NEXT: s_addc_u32 s5, s5, wobble at gotpcrel32@hi+12
-; GLOBALNESS0-NEXT: s_load_dwordx2 s[54:55], s[4:5], 0x0
+; GLOBALNESS0-NEXT: s_load_dwordx2 s[96:97], s[4:5], 0x0
; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], s[38:39]
; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], s[36:37]
-; GLOBALNESS0-NEXT: s_mov_b64 s[8:9], s[84:85]
+; GLOBALNESS0-NEXT: s_mov_b64 s[8:9], s[86:87]
; GLOBALNESS0-NEXT: s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS0-NEXT: s_mov_b32 s12, s82
-; GLOBALNESS0-NEXT: s_mov_b32 s13, s71
-; GLOBALNESS0-NEXT: s_mov_b32 s14, s70
+; GLOBALNESS0-NEXT: s_mov_b32 s12, s84
+; GLOBALNESS0-NEXT: s_mov_b32 s13, s83
+; GLOBALNESS0-NEXT: s_mov_b32 s14, s82
; GLOBALNESS0-NEXT: v_mov_b32_e32 v31, v41
; GLOBALNESS0-NEXT: s_waitcnt lgkmcnt(0)
-; GLOBALNESS0-NEXT: s_swappc_b64 s[30:31], s[54:55]
+; GLOBALNESS0-NEXT: s_swappc_b64 s[30:31], s[96:97]
; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], s[38:39]
; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], s[36:37]
-; GLOBALNESS0-NEXT: s_mov_b64 s[8:9], s[84:85]
+; GLOBALNESS0-NEXT: s_mov_b64 s[8:9], s[86:87]
; GLOBALNESS0-NEXT: s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS0-NEXT: s_mov_b32 s12, s82
-; GLOBALNESS0-NEXT: s_mov_b32 s13, s71
-; GLOBALNESS0-NEXT: s_mov_b32 s14, s70
+; GLOBALNESS0-NEXT: s_mov_b32 s12, s84
+; GLOBALNESS0-NEXT: s_mov_b32 s13, s83
+; GLOBALNESS0-NEXT: s_mov_b32 s14, s82
; GLOBALNESS0-NEXT: v_mov_b32_e32 v31, v41
; GLOBALNESS0-NEXT: global_store_dwordx2 v[44:45], v[58:59], off
; GLOBALNESS0-NEXT: ; implicit-def: $sgpr15
-; GLOBALNESS0-NEXT: s_swappc_b64 s[30:31], s[54:55]
-; GLOBALNESS0-NEXT: s_and_saveexec_b64 s[4:5], s[96:97]
+; GLOBALNESS0-NEXT: s_swappc_b64 s[30:31], s[96:97]
+; GLOBALNESS0-NEXT: s_and_saveexec_b64 s[4:5], s[70:71]
; GLOBALNESS0-NEXT: s_cbranch_execz .LBB1_14
; GLOBALNESS0-NEXT: ; %bb.23: ; %bb62.i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_16 Depth=2
@@ -593,22 +543,20 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: s_branch .LBB1_14
; GLOBALNESS0-NEXT: .LBB1_24: ; %Flow23
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT: v_readlane_b32 s84, v57, 8
-; GLOBALNESS0-NEXT: v_readlane_b32 s8, v57, 10
+; GLOBALNESS0-NEXT: v_readlane_b32 s8, v60, 8
; GLOBALNESS0-NEXT: v_pk_mov_b32 v[0:1], 0, 0
-; GLOBALNESS0-NEXT: s_mov_b32 s55, s83
-; GLOBALNESS0-NEXT: v_readlane_b32 s85, v57, 9
-; GLOBALNESS0-NEXT: v_readlane_b32 s9, v57, 11
+; GLOBALNESS0-NEXT: v_readlane_b32 s9, v60, 9
; GLOBALNESS0-NEXT: .LBB1_25: ; %Flow24
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT: s_or_b64 exec, exec, s[52:53]
-; GLOBALNESS0-NEXT: s_and_saveexec_b64 s[4:5], s[86:87]
+; GLOBALNESS0-NEXT: s_or_b64 exec, exec, s[98:99]
+; GLOBALNESS0-NEXT: s_and_saveexec_b64 s[4:5], s[68:69]
+; GLOBALNESS0-NEXT: s_load_dwordx4 s[68:71], s[48:49], 0x0
; GLOBALNESS0-NEXT: s_cbranch_execz .LBB1_2
; GLOBALNESS0-NEXT: ; %bb.26: ; %bb67.i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
-; GLOBALNESS0-NEXT: v_readlane_b32 s6, v57, 4
-; GLOBALNESS0-NEXT: v_readlane_b32 s7, v57, 5
-; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[6:7]
+; GLOBALNESS0-NEXT: v_readlane_b32 s6, v60, 4
+; GLOBALNESS0-NEXT: v_readlane_b32 s7, v60, 5
+; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[6:7]
; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_1
; GLOBALNESS0-NEXT: ; %bb.27: ; %bb69.i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
@@ -631,9 +579,9 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], s[38:39]
; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], s[36:37]
; GLOBALNESS0-NEXT: s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS0-NEXT: s_mov_b32 s12, s82
-; GLOBALNESS0-NEXT: s_mov_b32 s13, s71
-; GLOBALNESS0-NEXT: s_mov_b32 s14, s70
+; GLOBALNESS0-NEXT: s_mov_b32 s12, s84
+; GLOBALNESS0-NEXT: s_mov_b32 s13, s83
+; GLOBALNESS0-NEXT: s_mov_b32 s14, s82
; GLOBALNESS0-NEXT: v_mov_b32_e32 v31, v41
; GLOBALNESS0-NEXT: ; implicit-def: $sgpr15
; GLOBALNESS0-NEXT: s_swappc_b64 s[30:31], s[16:17]
@@ -650,9 +598,9 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], s[38:39]
; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], s[36:37]
; GLOBALNESS0-NEXT: s_mov_b64 s[10:11], s[34:35]
-; GLOBALNESS0-NEXT: s_mov_b32 s12, s82
-; GLOBALNESS0-NEXT: s_mov_b32 s13, s71
-; GLOBALNESS0-NEXT: s_mov_b32 s14, s70
+; GLOBALNESS0-NEXT: s_mov_b32 s12, s84
+; GLOBALNESS0-NEXT: s_mov_b32 s13, s83
+; GLOBALNESS0-NEXT: s_mov_b32 s14, s82
; GLOBALNESS0-NEXT: v_mov_b32_e32 v31, v41
; GLOBALNESS0-NEXT: ; implicit-def: $sgpr15
; GLOBALNESS0-NEXT: s_swappc_b64 s[30:31], s[16:17]
diff --git a/llvm/test/CodeGen/AMDGPU/v-cmp-cse-across-loop.mir b/llvm/test/CodeGen/AMDGPU/v-cmp-cse-across-loop.mir
new file mode 100644
index 0000000000000..3e366143958eb
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/v-cmp-cse-across-loop.mir
@@ -0,0 +1,67 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx1200 -mattr=+wavefrontsize64 -run-pass=machine-cse -verify-machineinstrs %s -o - | FileCheck %s
+# Check that V_CMP is not eliminated across the loop.
+
+---
+name: v_cmp_cse_across_loop
+tracksRegLiveness: true
+legalized: true
+regBankSelected: true
+selected: true
+body: |
+ ; CHECK-LABEL: name: v_cmp_cse_across_loop
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0, $sgpr0, $sgpr2_sgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sreg_64 = COPY $sgpr2_sgpr3
+ ; CHECK-NEXT: S_BRANCH %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: successors: %bb.1(0x40000000), %bb.2(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[V_CMP_GT_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_GT_U32_e64 [[COPY]], [[COPY1]], implicit $exec
+ ; CHECK-NEXT: [[SI_IF_BREAK:%[0-9]+]]:sreg_64 = SI_IF_BREAK killed [[V_CMP_GT_U32_e64_]], [[COPY2]], implicit-def dead $scc
+ ; CHECK-NEXT: SI_LOOP [[SI_IF_BREAK]], %bb.1, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: SI_END_CF [[SI_IF_BREAK]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+ ; CHECK-NEXT: [[V_CMP_GT_U32_e64_1:%[0-9]+]]:sreg_64 = V_CMP_GT_U32_e64 [[COPY]], [[COPY1]], implicit $exec
+ ; CHECK-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[V_CMP_GT_U32_e64_1]], $exec, implicit-def $scc
+ ; CHECK-NEXT: S_BRANCH %bb.3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ successors: %bb.1(0x80000000)
+ liveins: $vgpr0, $sgpr0, $sgpr2_sgpr3
+
+ %0:vgpr_32 = COPY $vgpr0
+ %1:sreg_32 = COPY $sgpr0
+ %2:sreg_64 = COPY $sgpr2_sgpr3
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.1(0x40000000), %bb.2(0x40000000)
+
+ %3:sreg_64 = V_CMP_GT_U32_e64 %0:vgpr_32, %1:sreg_32, implicit $exec
+ %4:sreg_64 = SI_IF_BREAK killed %3:sreg_64, %2:sreg_64, implicit-def dead $scc
+ SI_LOOP %4:sreg_64, %bb.1, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+ S_BRANCH %bb.2
+
+ bb.2:
+ successors: %bb.3(0x80000000)
+
+ SI_END_CF %4:sreg_64, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+ %5:sreg_64 = V_CMP_GT_U32_e64 %0:vgpr_32, %1:sreg_32, implicit $exec
+ %6:sreg_64 = S_AND_B64 %5:sreg_64, $exec, implicit-def $scc
+ S_BRANCH %bb.3
+
+ bb.3:
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll b/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
index a69dba90a9640..950a1252a3e06 100644
--- a/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
+++ b/llvm/test/CodeGen/AMDGPU/vgpr-descriptor-waterfall-loop-idom-update.ll
@@ -6,15 +6,15 @@ define void @vgpr_descriptor_waterfall_loop_idom_update(ptr %arg) #0 {
; GCN-LABEL: vgpr_descriptor_waterfall_loop_idom_update:
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_add_co_u32 v6, vcc_lo, v0, 8
-; GCN-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, 0, v1, vcc_lo
; GCN-NEXT: .LBB0_1: ; %bb0
; GCN-NEXT: ; =>This Loop Header: Depth=1
; GCN-NEXT: ; Child Loop BB0_2 Depth 2
+; GCN-NEXT: v_add_co_u32 v6, vcc_lo, v0, 8
+; GCN-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, 0, v1, vcc_lo
+; GCN-NEXT: s_mov_b32 s5, exec_lo
; GCN-NEXT: s_clause 0x1
; GCN-NEXT: flat_load_dwordx2 v[4:5], v[6:7]
; GCN-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; GCN-NEXT: s_mov_b32 s5, exec_lo
; GCN-NEXT: .LBB0_2: ; Parent Loop BB0_1 Depth=1
; GCN-NEXT: ; => This Inner Loop Header: Depth=2
; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
More information about the llvm-commits
mailing list